2024-11-07 12:35:57,119 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 12:35:57,130 main DEBUG Took 0.009452 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 12:35:57,130 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 12:35:57,130 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 12:35:57,131 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 12:35:57,132 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,139 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 12:35:57,151 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,152 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,153 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,154 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,155 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,156 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,157 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,157 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,158 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,159 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,159 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,160 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:35:57,161 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,161 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 12:35:57,162 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:35:57,163 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 12:35:57,165 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 12:35:57,165 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 12:35:57,166 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 12:35:57,167 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 12:35:57,175 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 12:35:57,177 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 12:35:57,179 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 12:35:57,179 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 12:35:57,179 main DEBUG createAppenders(={Console}) 2024-11-07 12:35:57,180 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-07 12:35:57,180 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 12:35:57,180 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-07 12:35:57,181 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 12:35:57,181 main DEBUG OutputStream closed 2024-11-07 12:35:57,181 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 12:35:57,182 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 12:35:57,182 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-07 12:35:57,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 12:35:57,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 12:35:57,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 12:35:57,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 12:35:57,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 12:35:57,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 12:35:57,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 12:35:57,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 12:35:57,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 12:35:57,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 12:35:57,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 12:35:57,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 12:35:57,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 12:35:57,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 12:35:57,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 12:35:57,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 12:35:57,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 12:35:57,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 12:35:57,258 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 12:35:57,258 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-07 12:35:57,258 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 12:35:57,259 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-07T12:35:57,479 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991 2024-11-07 12:35:57,482 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 12:35:57,483 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T12:35:57,491 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-07T12:35:57,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T12:35:57,529 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4, deleteOnExit=true 2024-11-07T12:35:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T12:35:57,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/test.cache.data in system properties and HBase conf 2024-11-07T12:35:57,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T12:35:57,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/hadoop.log.dir in system properties and HBase conf 2024-11-07T12:35:57,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T12:35:57,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T12:35:57,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T12:35:57,642 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T12:35:57,731 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T12:35:57,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T12:35:57,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T12:35:57,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T12:35:57,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T12:35:57,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T12:35:57,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T12:35:57,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T12:35:57,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T12:35:57,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T12:35:57,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/nfs.dump.dir in system properties and HBase conf 2024-11-07T12:35:57,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/java.io.tmpdir in system properties and HBase conf 2024-11-07T12:35:57,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T12:35:57,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T12:35:57,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T12:35:58,566 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T12:35:58,653 INFO [Time-limited test {}] log.Log(170): Logging initialized @2236ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T12:35:58,729 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:35:58,788 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T12:35:58,808 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T12:35:58,808 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T12:35:58,809 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T12:35:58,821 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:35:58,824 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/hadoop.log.dir/,AVAILABLE} 2024-11-07T12:35:58,825 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T12:35:59,024 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/java.io.tmpdir/jetty-localhost-33823-hadoop-hdfs-3_4_1-tests_jar-_-any-1430755928608819873/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T12:35:59,031 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33823} 2024-11-07T12:35:59,031 INFO [Time-limited test {}] server.Server(415): Started @2615ms 2024-11-07T12:35:59,413 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:35:59,419 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T12:35:59,420 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T12:35:59,421 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T12:35:59,421 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T12:35:59,423 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/hadoop.log.dir/,AVAILABLE} 2024-11-07T12:35:59,423 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T12:35:59,544 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/java.io.tmpdir/jetty-localhost-40123-hadoop-hdfs-3_4_1-tests_jar-_-any-14838051035186932758/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T12:35:59,545 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40123} 2024-11-07T12:35:59,545 INFO [Time-limited test {}] server.Server(415): Started @3129ms 2024-11-07T12:35:59,599 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T12:36:00,092 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/data/data2/current/BP-1511532250-172.17.0.3-1730982958310/current, will proceed with Du for space computation calculation, 2024-11-07T12:36:00,092 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/data/data1/current/BP-1511532250-172.17.0.3-1730982958310/current, will proceed with Du for space computation calculation, 2024-11-07T12:36:00,130 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T12:36:00,179 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb788dc31dc0a5a71 with lease ID 0x237dd3c51bf1b45: Processing first storage report for DS-d4910f68-39d8-492f-9d6a-0df14bfdc14a from datanode DatanodeRegistration(127.0.0.1:45087, datanodeUuid=0ac894f8-e0ee-43f1-9f78-950a1864e70d, infoPort=44899, infoSecurePort=0, ipcPort=34585, storageInfo=lv=-57;cid=testClusterID;nsid=670655866;c=1730982958310) 2024-11-07T12:36:00,181 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb788dc31dc0a5a71 with lease ID 0x237dd3c51bf1b45: from storage DS-d4910f68-39d8-492f-9d6a-0df14bfdc14a node DatanodeRegistration(127.0.0.1:45087, datanodeUuid=0ac894f8-e0ee-43f1-9f78-950a1864e70d, infoPort=44899, infoSecurePort=0, ipcPort=34585, storageInfo=lv=-57;cid=testClusterID;nsid=670655866;c=1730982958310), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T12:36:00,181 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb788dc31dc0a5a71 with lease ID 0x237dd3c51bf1b45: Processing first storage report for DS-1a803d8d-c51a-4542-8613-c196bcb43f69 from datanode DatanodeRegistration(127.0.0.1:45087, datanodeUuid=0ac894f8-e0ee-43f1-9f78-950a1864e70d, infoPort=44899, infoSecurePort=0, ipcPort=34585, storageInfo=lv=-57;cid=testClusterID;nsid=670655866;c=1730982958310) 2024-11-07T12:36:00,181 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb788dc31dc0a5a71 with lease ID 0x237dd3c51bf1b45: from storage DS-1a803d8d-c51a-4542-8613-c196bcb43f69 node DatanodeRegistration(127.0.0.1:45087, datanodeUuid=0ac894f8-e0ee-43f1-9f78-950a1864e70d, infoPort=44899, infoSecurePort=0, ipcPort=34585, storageInfo=lv=-57;cid=testClusterID;nsid=670655866;c=1730982958310), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T12:36:00,249 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991 2024-11-07T12:36:00,322 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/zookeeper_0, clientPort=62093, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T12:36:00,332 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62093 2024-11-07T12:36:00,342 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:36:00,345 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:36:00,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741825_1001 (size=7) 2024-11-07T12:36:00,968 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e with version=8 2024-11-07T12:36:00,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/hbase-staging 2024-11-07T12:36:01,063 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T12:36:01,312 INFO [Time-limited test {}] client.ConnectionUtils(128): master/87fb68174c52:0 server-side Connection retries=45 2024-11-07T12:36:01,323 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:36:01,323 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T12:36:01,328 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T12:36:01,328 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:36:01,328 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T12:36:01,464 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T12:36:01,527 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T12:36:01,536 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T12:36:01,541 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T12:36:01,568 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 6481 (auto-detected) 2024-11-07T12:36:01,569 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-07T12:36:01,587 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36251 2024-11-07T12:36:01,609 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36251 connecting to ZooKeeper ensemble=127.0.0.1:62093 2024-11-07T12:36:01,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:362510x0, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T12:36:01,643 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36251-0x1001a3de74f0000 connected 2024-11-07T12:36:01,667 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:36:01,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:36:01,678 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:36:01,682 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e, hbase.cluster.distributed=false 2024-11-07T12:36:01,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T12:36:01,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36251 2024-11-07T12:36:01,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36251 2024-11-07T12:36:01,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36251 2024-11-07T12:36:01,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36251 2024-11-07T12:36:01,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36251 2024-11-07T12:36:01,821 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/87fb68174c52:0 server-side Connection retries=45 2024-11-07T12:36:01,823 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:36:01,823 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T12:36:01,824 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T12:36:01,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:36:01,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T12:36:01,827 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T12:36:01,829 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T12:36:01,830 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38515 2024-11-07T12:36:01,832 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38515 connecting to ZooKeeper ensemble=127.0.0.1:62093 2024-11-07T12:36:01,833 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:36:01,836 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:36:01,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385150x0, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T12:36:01,843 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38515-0x1001a3de74f0001 connected 2024-11-07T12:36:01,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:36:01,848 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T12:36:01,858 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T12:36:01,861 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T12:36:01,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T12:36:01,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38515 2024-11-07T12:36:01,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38515 2024-11-07T12:36:01,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38515 2024-11-07T12:36:01,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38515 2024-11-07T12:36:01,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38515 2024-11-07T12:36:01,887 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;87fb68174c52:36251 2024-11-07T12:36:01,888 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/87fb68174c52,36251,1730982961115 2024-11-07T12:36:01,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:36:01,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:36:01,897 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/87fb68174c52,36251,1730982961115 2024-11-07T12:36:01,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T12:36:01,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:01,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:01,919 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T12:36:01,921 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/87fb68174c52,36251,1730982961115 from backup master directory 2024-11-07T12:36:01,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/87fb68174c52,36251,1730982961115 2024-11-07T12:36:01,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:36:01,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:36:01,925 WARN [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T12:36:01,926 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=87fb68174c52,36251,1730982961115 2024-11-07T12:36:01,928 INFO [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T12:36:01,930 INFO [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T12:36:01,990 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/hbase.id] with ID: 55d9ff63-aba5-4d8c-b02d-9ea864170a03 2024-11-07T12:36:01,990 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/.tmp/hbase.id 2024-11-07T12:36:02,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741826_1002 (size=42) 2024-11-07T12:36:02,403 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/.tmp/hbase.id]:[hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/hbase.id] 2024-11-07T12:36:02,449 INFO [master/87fb68174c52:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:36:02,454 INFO [master/87fb68174c52:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T12:36:02,473 INFO [master/87fb68174c52:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-07T12:36:02,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:02,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:02,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741827_1003 (size=196) 2024-11-07T12:36:02,507 INFO [master/87fb68174c52:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:36:02,510 INFO [master/87fb68174c52:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T12:36:02,527 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:02,532 INFO [master/87fb68174c52:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:36:02,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741828_1004 (size=1189) 2024-11-07T12:36:02,982 INFO [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store 2024-11-07T12:36:03,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741829_1005 (size=34) 2024-11-07T12:36:03,408 INFO [master/87fb68174c52:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T12:36:03,411 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:03,412 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T12:36:03,413 INFO [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:36:03,413 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:36:03,414 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T12:36:03,414 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:36:03,415 INFO [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:36:03,416 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730982963412Disabling compacts and flushes for region at 1730982963412Disabling writes for close at 1730982963414 (+2 ms)Writing region close event to WAL at 1730982963415 (+1 ms)Closed at 1730982963415 2024-11-07T12:36:03,418 WARN [master/87fb68174c52:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/.initializing 2024-11-07T12:36:03,419 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/WALs/87fb68174c52,36251,1730982961115 2024-11-07T12:36:03,427 INFO [master/87fb68174c52:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T12:36:03,443 INFO [master/87fb68174c52:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87fb68174c52%2C36251%2C1730982961115, suffix=, logDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/WALs/87fb68174c52,36251,1730982961115, archiveDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/oldWALs, maxLogs=10 2024-11-07T12:36:03,476 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/WALs/87fb68174c52,36251,1730982961115/87fb68174c52%2C36251%2C1730982961115.1730982963449, exclude list is [], retry=0 2024-11-07T12:36:03,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45087,DS-d4910f68-39d8-492f-9d6a-0df14bfdc14a,DISK] 2024-11-07T12:36:03,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T12:36:03,529 INFO [master/87fb68174c52:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/WALs/87fb68174c52,36251,1730982961115/87fb68174c52%2C36251%2C1730982961115.1730982963449 2024-11-07T12:36:03,530 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44899:44899)] 2024-11-07T12:36:03,531 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:36:03,531 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:03,534 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,535 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,597 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T12:36:03,601 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:03,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:03,604 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T12:36:03,607 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:03,608 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:03,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T12:36:03,611 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:03,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:03,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T12:36:03,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:03,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:03,617 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,621 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,622 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,627 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,628 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,631 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T12:36:03,634 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:36:03,639 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:36:03,640 INFO [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60215099, jitterRate=-0.10272510349750519}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T12:36:03,645 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730982963547Initializing all the Stores at 1730982963550 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982963550Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982963551 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982963551Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982963551Cleaning up temporary data from old regions at 1730982963628 (+77 ms)Region opened successfully at 1730982963645 (+17 ms) 2024-11-07T12:36:03,646 INFO [master/87fb68174c52:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T12:36:03,678 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a46cf2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87fb68174c52/172.17.0.3:0 2024-11-07T12:36:03,710 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T12:36:03,721 INFO [master/87fb68174c52:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T12:36:03,721 INFO [master/87fb68174c52:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T12:36:03,724 INFO [master/87fb68174c52:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T12:36:03,726 INFO [master/87fb68174c52:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T12:36:03,730 INFO [master/87fb68174c52:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T12:36:03,731 INFO [master/87fb68174c52:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T12:36:03,760 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T12:36:03,770 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T12:36:03,772 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T12:36:03,775 INFO [master/87fb68174c52:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T12:36:03,776 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T12:36:03,778 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T12:36:03,780 INFO [master/87fb68174c52:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T12:36:03,783 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T12:36:03,784 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T12:36:03,786 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T12:36:03,788 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T12:36:03,805 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T12:36:03,807 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T12:36:03,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T12:36:03,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T12:36:03,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:03,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:03,813 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=87fb68174c52,36251,1730982961115, sessionid=0x1001a3de74f0000, setting cluster-up flag (Was=false) 2024-11-07T12:36:03,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:03,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:03,831 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T12:36:03,833 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=87fb68174c52,36251,1730982961115 2024-11-07T12:36:03,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:03,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:03,844 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T12:36:03,845 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=87fb68174c52,36251,1730982961115 2024-11-07T12:36:03,851 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T12:36:03,875 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(746): ClusterId : 55d9ff63-aba5-4d8c-b02d-9ea864170a03 2024-11-07T12:36:03,878 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T12:36:03,883 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T12:36:03,883 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T12:36:03,886 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T12:36:03,887 DEBUG [RS:0;87fb68174c52:38515 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e777bae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87fb68174c52/172.17.0.3:0 2024-11-07T12:36:03,901 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;87fb68174c52:38515 2024-11-07T12:36:03,904 INFO [RS:0;87fb68174c52:38515 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T12:36:03,905 INFO [RS:0;87fb68174c52:38515 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T12:36:03,905 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T12:36:03,907 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(2659): reportForDuty to master=87fb68174c52,36251,1730982961115 with port=38515, startcode=1730982961782 2024-11-07T12:36:03,919 DEBUG [RS:0;87fb68174c52:38515 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T12:36:03,924 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T12:36:03,936 INFO [master/87fb68174c52:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T12:36:03,948 INFO [master/87fb68174c52:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T12:36:03,956 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54785, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T12:36:03,953 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 87fb68174c52,36251,1730982961115 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T12:36:03,961 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/87fb68174c52:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:36:03,961 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/87fb68174c52:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:36:03,961 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/87fb68174c52:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:36:03,961 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:03,962 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/87fb68174c52:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:36:03,962 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/87fb68174c52:0, corePoolSize=10, maxPoolSize=10 2024-11-07T12:36:03,962 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:03,962 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/87fb68174c52:0, corePoolSize=2, maxPoolSize=2 2024-11-07T12:36:03,962 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:03,963 INFO [master/87fb68174c52:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730982993963 2024-11-07T12:36:03,965 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T12:36:03,966 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T12:36:03,967 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:36:03,968 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T12:36:03,970 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T12:36:03,970 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T12:36:03,971 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T12:36:03,971 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T12:36:03,972 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:03,975 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T12:36:03,975 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:03,975 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T12:36:03,976 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T12:36:03,977 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T12:36:03,980 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T12:36:03,981 INFO [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T12:36:03,983 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/87fb68174c52:0:becomeActiveMaster-HFileCleaner.large.0-1730982963982,5,FailOnTimeoutGroup] 2024-11-07T12:36:03,985 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/87fb68174c52:0:becomeActiveMaster-HFileCleaner.small.0-1730982963983,5,FailOnTimeoutGroup] 2024-11-07T12:36:03,985 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:03,985 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T12:36:03,986 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:03,987 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:03,988 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-07T12:36:03,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741831_1007 (size=1321) 2024-11-07T12:36:03,988 WARN [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-07T12:36:04,089 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(2659): reportForDuty to master=87fb68174c52,36251,1730982961115 with port=38515, startcode=1730982961782 2024-11-07T12:36:04,091 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36251 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 87fb68174c52,38515,1730982961782 2024-11-07T12:36:04,094 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36251 {}] master.ServerManager(517): Registering regionserver=87fb68174c52,38515,1730982961782 2024-11-07T12:36:04,102 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e 2024-11-07T12:36:04,103 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43681 2024-11-07T12:36:04,103 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T12:36:04,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T12:36:04,107 DEBUG [RS:0;87fb68174c52:38515 {}] zookeeper.ZKUtil(111): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/87fb68174c52,38515,1730982961782 2024-11-07T12:36:04,107 WARN [RS:0;87fb68174c52:38515 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T12:36:04,108 INFO [RS:0;87fb68174c52:38515 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:36:04,108 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/WALs/87fb68174c52,38515,1730982961782 2024-11-07T12:36:04,109 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [87fb68174c52,38515,1730982961782] 2024-11-07T12:36:04,133 INFO [RS:0;87fb68174c52:38515 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T12:36:04,147 INFO [RS:0;87fb68174c52:38515 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T12:36:04,152 INFO [RS:0;87fb68174c52:38515 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T12:36:04,152 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,153 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T12:36:04,158 INFO [RS:0;87fb68174c52:38515 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T12:36:04,160 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,160 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,160 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,160 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,160 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,161 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,161 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/87fb68174c52:0, corePoolSize=2, maxPoolSize=2 2024-11-07T12:36:04,161 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,161 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,161 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,162 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,162 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,162 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/87fb68174c52:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:36:04,162 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/87fb68174c52:0, corePoolSize=3, maxPoolSize=3 2024-11-07T12:36:04,163 DEBUG [RS:0;87fb68174c52:38515 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0, corePoolSize=3, maxPoolSize=3 2024-11-07T12:36:04,165 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,165 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,165 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,165 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,165 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,165 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb68174c52,38515,1730982961782-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T12:36:04,185 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T12:36:04,187 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb68174c52,38515,1730982961782-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,187 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,187 INFO [RS:0;87fb68174c52:38515 {}] regionserver.Replication(171): 87fb68174c52,38515,1730982961782 started 2024-11-07T12:36:04,206 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:04,206 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1482): Serving as 87fb68174c52,38515,1730982961782, RpcServer on 87fb68174c52/172.17.0.3:38515, sessionid=0x1001a3de74f0001 2024-11-07T12:36:04,207 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T12:36:04,207 DEBUG [RS:0;87fb68174c52:38515 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 87fb68174c52,38515,1730982961782 2024-11-07T12:36:04,207 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87fb68174c52,38515,1730982961782' 2024-11-07T12:36:04,207 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T12:36:04,209 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T12:36:04,209 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T12:36:04,209 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T12:36:04,209 DEBUG [RS:0;87fb68174c52:38515 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 87fb68174c52,38515,1730982961782 2024-11-07T12:36:04,209 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87fb68174c52,38515,1730982961782' 2024-11-07T12:36:04,210 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T12:36:04,210 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T12:36:04,211 DEBUG [RS:0;87fb68174c52:38515 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T12:36:04,211 INFO [RS:0;87fb68174c52:38515 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T12:36:04,211 INFO [RS:0;87fb68174c52:38515 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T12:36:04,316 INFO [RS:0;87fb68174c52:38515 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T12:36:04,320 INFO [RS:0;87fb68174c52:38515 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87fb68174c52%2C38515%2C1730982961782, suffix=, logDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/WALs/87fb68174c52,38515,1730982961782, archiveDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/oldWALs, maxLogs=32 2024-11-07T12:36:04,337 DEBUG [RS:0;87fb68174c52:38515 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/WALs/87fb68174c52,38515,1730982961782/87fb68174c52%2C38515%2C1730982961782.1730982964322, exclude list is [], retry=0 2024-11-07T12:36:04,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45087,DS-d4910f68-39d8-492f-9d6a-0df14bfdc14a,DISK] 2024-11-07T12:36:04,344 INFO [RS:0;87fb68174c52:38515 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/WALs/87fb68174c52,38515,1730982961782/87fb68174c52%2C38515%2C1730982961782.1730982964322 2024-11-07T12:36:04,345 DEBUG [RS:0;87fb68174c52:38515 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44899:44899)] 2024-11-07T12:36:04,390 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T12:36:04,391 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e 2024-11-07T12:36:04,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741833_1009 (size=32) 2024-11-07T12:36:04,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:04,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T12:36:04,807 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T12:36:04,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:04,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:04,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T12:36:04,811 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T12:36:04,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:04,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:04,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T12:36:04,814 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T12:36:04,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:04,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:04,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T12:36:04,817 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T12:36:04,818 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:04,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:04,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T12:36:04,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740 2024-11-07T12:36:04,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740 2024-11-07T12:36:04,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T12:36:04,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T12:36:04,824 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:36:04,826 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T12:36:04,830 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:36:04,831 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67094941, jitterRate=-2.0746886730194092E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:36:04,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730982964802Initializing all the Stores at 1730982964804 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982964804Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982964805 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982964805Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982964805Cleaning up temporary data from old regions at 1730982964823 (+18 ms)Region opened successfully at 1730982964833 (+10 ms) 2024-11-07T12:36:04,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T12:36:04,833 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T12:36:04,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T12:36:04,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T12:36:04,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T12:36:04,835 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T12:36:04,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730982964833Disabling compacts and flushes for region at 1730982964833Disabling writes for close at 1730982964834 (+1 ms)Writing region close event to WAL at 1730982964835 (+1 ms)Closed at 1730982964835 2024-11-07T12:36:04,838 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:36:04,838 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T12:36:04,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T12:36:04,853 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T12:36:04,855 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T12:36:05,008 DEBUG [87fb68174c52:36251 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T12:36:05,020 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:05,027 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87fb68174c52,38515,1730982961782, state=OPENING 2024-11-07T12:36:05,032 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T12:36:05,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:05,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:36:05,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:36:05,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:36:05,036 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T12:36:05,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:36:05,212 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T12:36:05,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58087, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T12:36:05,227 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T12:36:05,227 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:36:05,228 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T12:36:05,231 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87fb68174c52%2C38515%2C1730982961782.meta, suffix=.meta, logDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/WALs/87fb68174c52,38515,1730982961782, archiveDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/oldWALs, maxLogs=32 2024-11-07T12:36:05,247 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/WALs/87fb68174c52,38515,1730982961782/87fb68174c52%2C38515%2C1730982961782.meta.1730982965232.meta, exclude list is [], retry=0 2024-11-07T12:36:05,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45087,DS-d4910f68-39d8-492f-9d6a-0df14bfdc14a,DISK] 2024-11-07T12:36:05,253 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/WALs/87fb68174c52,38515,1730982961782/87fb68174c52%2C38515%2C1730982961782.meta.1730982965232.meta 2024-11-07T12:36:05,254 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44899:44899)] 2024-11-07T12:36:05,254 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:36:05,256 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T12:36:05,258 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T12:36:05,263 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T12:36:05,267 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T12:36:05,268 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:05,268 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T12:36:05,268 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T12:36:05,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T12:36:05,272 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T12:36:05,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:05,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:05,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T12:36:05,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T12:36:05,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:05,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:05,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T12:36:05,278 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T12:36:05,278 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:05,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:05,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T12:36:05,280 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T12:36:05,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:05,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:36:05,281 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T12:36:05,283 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740 2024-11-07T12:36:05,285 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740 2024-11-07T12:36:05,287 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T12:36:05,288 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T12:36:05,288 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:36:05,291 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T12:36:05,292 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69272214, jitterRate=0.03223642706871033}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:36:05,292 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T12:36:05,294 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730982965268Writing region info on filesystem at 1730982965269 (+1 ms)Initializing all the Stores at 1730982965270 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982965270Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982965271 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982965271Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982965271Cleaning up temporary data from old regions at 1730982965288 (+17 ms)Running coprocessor post-open hooks at 1730982965292 (+4 ms)Region opened successfully at 1730982965294 (+2 ms) 2024-11-07T12:36:05,301 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730982965203 2024-11-07T12:36:05,312 DEBUG [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T12:36:05,312 INFO [RS_OPEN_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T12:36:05,314 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:05,316 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87fb68174c52,38515,1730982961782, state=OPEN 2024-11-07T12:36:05,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T12:36:05,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T12:36:05,320 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:36:05,320 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:36:05,320 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:05,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T12:36:05,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87fb68174c52,38515,1730982961782 in 282 msec 2024-11-07T12:36:05,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T12:36:05,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 482 msec 2024-11-07T12:36:05,333 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:36:05,333 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T12:36:05,353 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:05,354 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:05,375 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:05,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44289, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:05,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5240 sec 2024-11-07T12:36:05,405 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730982965404, completionTime=-1 2024-11-07T12:36:05,408 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T12:36:05,409 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T12:36:05,439 INFO [master/87fb68174c52:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-07T12:36:05,439 INFO [master/87fb68174c52:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730983025439 2024-11-07T12:36:05,440 INFO [master/87fb68174c52:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730983085439 2024-11-07T12:36:05,440 INFO [master/87fb68174c52:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-07T12:36:05,442 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb68174c52,36251,1730982961115-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:05,443 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb68174c52,36251,1730982961115-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:05,443 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb68174c52,36251,1730982961115-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:05,445 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-87fb68174c52:36251, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:05,445 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:05,446 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:05,451 DEBUG [master/87fb68174c52:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T12:36:05,476 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.549sec 2024-11-07T12:36:05,477 INFO [master/87fb68174c52:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T12:36:05,479 INFO [master/87fb68174c52:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T12:36:05,480 INFO [master/87fb68174c52:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T12:36:05,481 INFO [master/87fb68174c52:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T12:36:05,481 INFO [master/87fb68174c52:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T12:36:05,482 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb68174c52,36251,1730982961115-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T12:36:05,482 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb68174c52,36251,1730982961115-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T12:36:05,491 DEBUG [master/87fb68174c52:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T12:36:05,492 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T12:36:05,492 INFO [master/87fb68174c52:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87fb68174c52,36251,1730982961115-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:36:05,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:05,588 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T12:36:05,588 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T12:36:05,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:05,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:05,602 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39400f33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:05,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:05,611 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:05,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:05,614 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:05,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:05,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=87fb68174c52,36251,1730982961115 2024-11-07T12:36:05,655 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=310, ProcessCount=11, AvailableMemoryMB=6725 2024-11-07T12:36:05,685 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:36:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb68174c52,36251,1730982961115 2024-11-07T12:36:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7384b5b5 2024-11-07T12:36:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:36:05,697 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:36:05,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:36:05,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:36:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:05,718 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:36:05,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-07T12:36:05,721 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:05,723 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:36:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:36:05,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741835_1011 (size=959) 2024-11-07T12:36:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:36:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:36:06,137 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e 2024-11-07T12:36:06,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741836_1012 (size=53) 2024-11-07T12:36:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:36:06,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:06,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f7b6889d440fde956272a8aab00142f0, disabling compactions & flushes 2024-11-07T12:36:06,547 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:06,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:06,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. after waiting 0 ms 2024-11-07T12:36:06,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:06,547 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:06,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f7b6889d440fde956272a8aab00142f0: Waiting for close lock at 1730982966547Disabling compacts and flushes for region at 1730982966547Disabling writes for close at 1730982966547Writing region close event to WAL at 1730982966547Closed at 1730982966547 2024-11-07T12:36:06,550 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:36:06,555 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730982966550"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982966550"}]},"ts":"1730982966550"} 2024-11-07T12:36:06,559 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:36:06,561 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:36:06,564 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982966561"}]},"ts":"1730982966561"} 2024-11-07T12:36:06,568 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:36:06,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, ASSIGN}] 2024-11-07T12:36:06,571 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, ASSIGN 2024-11-07T12:36:06,573 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, ASSIGN; state=OFFLINE, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=false 2024-11-07T12:36:06,725 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f7b6889d440fde956272a8aab00142f0, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:06,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, ASSIGN because future has completed 2024-11-07T12:36:06,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:36:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:36:06,891 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:06,892 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:36:06,892 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,892 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:06,893 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,893 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,895 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,897 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7b6889d440fde956272a8aab00142f0 columnFamilyName A 2024-11-07T12:36:06,897 DEBUG [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:06,898 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] regionserver.HStore(327): Store=f7b6889d440fde956272a8aab00142f0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:06,898 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,901 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7b6889d440fde956272a8aab00142f0 columnFamilyName B 2024-11-07T12:36:06,901 DEBUG [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:06,901 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] regionserver.HStore(327): Store=f7b6889d440fde956272a8aab00142f0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:06,902 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,904 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7b6889d440fde956272a8aab00142f0 columnFamilyName C 2024-11-07T12:36:06,904 DEBUG [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:06,905 INFO [StoreOpener-f7b6889d440fde956272a8aab00142f0-1 {}] regionserver.HStore(327): Store=f7b6889d440fde956272a8aab00142f0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:06,905 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,907 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,907 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,910 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,910 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,911 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:36:06,914 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,917 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:36:06,918 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f7b6889d440fde956272a8aab00142f0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63365701, jitterRate=-0.055777475237846375}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:36:06,918 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:06,919 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f7b6889d440fde956272a8aab00142f0: Running coprocessor pre-open hook at 1730982966893Writing region info on filesystem at 1730982966893Initializing all the Stores at 1730982966894 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982966894Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982966894Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982966895 (+1 ms)Cleaning up temporary data from old regions at 1730982966910 (+15 ms)Running coprocessor post-open hooks at 1730982966918 (+8 ms)Region opened successfully at 1730982966919 (+1 ms) 2024-11-07T12:36:06,921 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., pid=6, masterSystemTime=1730982966884 2024-11-07T12:36:06,924 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:06,924 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:06,926 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f7b6889d440fde956272a8aab00142f0, regionState=OPEN, openSeqNum=2, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:06,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:36:06,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T12:36:06,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 in 202 msec 2024-11-07T12:36:06,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T12:36:06,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, ASSIGN in 366 msec 2024-11-07T12:36:06,941 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:36:06,941 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982966941"}]},"ts":"1730982966941"} 2024-11-07T12:36:06,944 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:36:06,946 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:36:06,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2350 sec 2024-11-07T12:36:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:36:07,879 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:07,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31097510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,889 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff1be82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,897 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:36:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb68174c52,36251,1730982961115 2024-11-07T12:36:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@254d7cb 2024-11-07T12:36:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:36:07,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:36:07,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,908 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591de3bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,911 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcb1101, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,914 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f7011d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,917 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6488fd9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,921 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fc0926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,924 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4474c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,928 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ffac79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,930 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5580124c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,934 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3449e9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,936 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b88104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,939 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cbeb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,942 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6435e78b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,945 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce91b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,947 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b16cd13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,952 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbc32ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,954 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f894d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:07,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:07,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b1e9af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:07,959 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181c6821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:07,965 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,965 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,965 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,965 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,965 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,967 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,967 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,971 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,971 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,973 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,981 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:07,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:07,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:36:07,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-07T12:36:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:36:08,008 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:08,010 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:08,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:08,055 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:08,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:36:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:36:08,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3c4ef4e42cae41538f9ee1048159479f is 50, key is test_row_0/A:col10/1730982968043/Put/seqid=0 2024-11-07T12:36:08,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:36:08,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:08,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741837_1013 (size=12001) 2024-11-07T12:36:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983028221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983028218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983028229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983028229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983028234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:36:08,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983028350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983028348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983028350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983028350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983028351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:36:08,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:08,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:36:08,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:08,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983028559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983028560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983028560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983028569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983028569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3c4ef4e42cae41538f9ee1048159479f 2024-11-07T12:36:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:36:08,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:36:08,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:08,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/b55e24593c4d47bf86ff8dd3df51b63b is 50, key is test_row_0/B:col10/1730982968043/Put/seqid=0 2024-11-07T12:36:08,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741838_1014 (size=12001) 2024-11-07T12:36:08,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/b55e24593c4d47bf86ff8dd3df51b63b 2024-11-07T12:36:08,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/0912bd9049db4378aa488e85ce6e79ba is 50, key is test_row_0/C:col10/1730982968043/Put/seqid=0 2024-11-07T12:36:08,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741839_1015 (size=12001) 2024-11-07T12:36:08,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/0912bd9049db4378aa488e85ce6e79ba 2024-11-07T12:36:08,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:36:08,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:08,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:08,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:08,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3c4ef4e42cae41538f9ee1048159479f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3c4ef4e42cae41538f9ee1048159479f 2024-11-07T12:36:08,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3c4ef4e42cae41538f9ee1048159479f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T12:36:08,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/b55e24593c4d47bf86ff8dd3df51b63b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b55e24593c4d47bf86ff8dd3df51b63b 2024-11-07T12:36:08,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b55e24593c4d47bf86ff8dd3df51b63b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T12:36:08,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/0912bd9049db4378aa488e85ce6e79ba as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/0912bd9049db4378aa488e85ce6e79ba 2024-11-07T12:36:08,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983028869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983028869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/0912bd9049db4378aa488e85ce6e79ba, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T12:36:08,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7b6889d440fde956272a8aab00142f0 in 850ms, sequenceid=14, compaction requested=false 2024-11-07T12:36:08,918 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T12:36:08,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:08,923 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:08,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T12:36:08,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/0ff3c41962cb4ad387a6cd00e427dfdb is 50, key is test_row_0/A:col10/1730982968924/Put/seqid=0 2024-11-07T12:36:08,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983028961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983028965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:08,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983028972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741840_1016 (size=23705) 2024-11-07T12:36:09,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/0ff3c41962cb4ad387a6cd00e427dfdb 2024-11-07T12:36:09,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:36:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:09,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:09,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/b9b8a15a556442ccbf4e4773497e334b is 50, key is test_row_0/B:col10/1730982968924/Put/seqid=0 2024-11-07T12:36:09,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741841_1017 (size=12001) 2024-11-07T12:36:09,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/b9b8a15a556442ccbf4e4773497e334b 2024-11-07T12:36:09,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983029079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983029080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c1201e51bfb644e2bb8faad993eddcca is 50, key is test_row_0/C:col10/1730982968924/Put/seqid=0 2024-11-07T12:36:09,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983029088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741842_1018 (size=12001) 2024-11-07T12:36:09,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c1201e51bfb644e2bb8faad993eddcca 2024-11-07T12:36:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:36:09,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:36:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:09,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:09,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:09,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/0ff3c41962cb4ad387a6cd00e427dfdb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/0ff3c41962cb4ad387a6cd00e427dfdb 2024-11-07T12:36:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:09,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/0ff3c41962cb4ad387a6cd00e427dfdb, entries=400, sequenceid=42, filesize=23.1 K 2024-11-07T12:36:09,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/b9b8a15a556442ccbf4e4773497e334b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b9b8a15a556442ccbf4e4773497e334b 2024-11-07T12:36:09,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b9b8a15a556442ccbf4e4773497e334b, entries=150, sequenceid=42, filesize=11.7 K 2024-11-07T12:36:09,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c1201e51bfb644e2bb8faad993eddcca as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c1201e51bfb644e2bb8faad993eddcca 2024-11-07T12:36:09,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c1201e51bfb644e2bb8faad993eddcca, entries=150, sequenceid=42, filesize=11.7 K 2024-11-07T12:36:09,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for f7b6889d440fde956272a8aab00142f0 in 321ms, sequenceid=42, compaction requested=false 2024-11-07T12:36:09,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:09,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:36:09,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:09,320 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T12:36:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:09,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/55cae0d8207546609bc3899ba0ad5cb5 is 50, key is test_row_0/A:col10/1730982968964/Put/seqid=0 2024-11-07T12:36:09,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741843_1019 (size=16681) 2024-11-07T12:36:09,375 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/55cae0d8207546609bc3899ba0ad5cb5 2024-11-07T12:36:09,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3276cda3d08f46609c83cd7afaf47cb1 is 50, key is test_row_0/B:col10/1730982968964/Put/seqid=0 2024-11-07T12:36:09,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741844_1020 (size=12001) 2024-11-07T12:36:09,444 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3276cda3d08f46609c83cd7afaf47cb1 2024-11-07T12:36:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983029435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983029435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983029438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983029449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983029451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/8946d2ee109349a0986ba2f15b6d3c11 is 50, key is test_row_0/C:col10/1730982968964/Put/seqid=0 2024-11-07T12:36:09,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741845_1021 (size=12001) 2024-11-07T12:36:09,502 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/8946d2ee109349a0986ba2f15b6d3c11 2024-11-07T12:36:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/55cae0d8207546609bc3899ba0ad5cb5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55cae0d8207546609bc3899ba0ad5cb5 2024-11-07T12:36:09,531 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55cae0d8207546609bc3899ba0ad5cb5, entries=250, sequenceid=52, filesize=16.3 K 2024-11-07T12:36:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3276cda3d08f46609c83cd7afaf47cb1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3276cda3d08f46609c83cd7afaf47cb1 2024-11-07T12:36:09,547 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3276cda3d08f46609c83cd7afaf47cb1, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T12:36:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/8946d2ee109349a0986ba2f15b6d3c11 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/8946d2ee109349a0986ba2f15b6d3c11 2024-11-07T12:36:09,561 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/8946d2ee109349a0986ba2f15b6d3c11, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T12:36:09,564 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for f7b6889d440fde956272a8aab00142f0 in 243ms, sequenceid=52, compaction requested=true 2024-11-07T12:36:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:09,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:09,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-07T12:36:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:09,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T12:36:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-07T12:36:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:09,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T12:36:09,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-07T12:36:09,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5770 sec 2024-11-07T12:36:09,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3298618afc5849bb8ffbeac075f76961 is 50, key is test_row_0/A:col10/1730982969444/Put/seqid=0 2024-11-07T12:36:09,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983029580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983029592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983029593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983029597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983029597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741846_1022 (size=14341) 2024-11-07T12:36:09,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983029708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983029717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983029718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983029718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983029729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983029917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983029938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983029938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983029939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:09,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983029949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3298618afc5849bb8ffbeac075f76961 2024-11-07T12:36:10,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/519d18c65f684e818dfeca2ee0d2535c is 50, key is test_row_0/B:col10/1730982969444/Put/seqid=0 2024-11-07T12:36:10,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741847_1023 (size=12001) 2024-11-07T12:36:10,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/519d18c65f684e818dfeca2ee0d2535c 2024-11-07T12:36:10,127 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T12:36:10,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/fcf5c163b42541a5ab2dc11526f7b423 is 50, key is test_row_0/C:col10/1730982969444/Put/seqid=0 2024-11-07T12:36:10,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741848_1024 (size=12001) 2024-11-07T12:36:10,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/fcf5c163b42541a5ab2dc11526f7b423 2024-11-07T12:36:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:36:10,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:10,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-07T12:36:10,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3298618afc5849bb8ffbeac075f76961 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3298618afc5849bb8ffbeac075f76961 2024-11-07T12:36:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:36:10,172 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:10,174 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:10,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:10,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3298618afc5849bb8ffbeac075f76961, entries=200, sequenceid=79, filesize=14.0 K 2024-11-07T12:36:10,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/519d18c65f684e818dfeca2ee0d2535c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/519d18c65f684e818dfeca2ee0d2535c 2024-11-07T12:36:10,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/519d18c65f684e818dfeca2ee0d2535c, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T12:36:10,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/fcf5c163b42541a5ab2dc11526f7b423 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/fcf5c163b42541a5ab2dc11526f7b423 2024-11-07T12:36:10,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/fcf5c163b42541a5ab2dc11526f7b423, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T12:36:10,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for f7b6889d440fde956272a8aab00142f0 in 658ms, sequenceid=79, compaction requested=true 2024-11-07T12:36:10,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:10,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:10,238 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:10,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:10,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:10,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:10,239 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:10,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:10,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:10,245 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:10,247 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:10,247 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,248 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b55e24593c4d47bf86ff8dd3df51b63b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b9b8a15a556442ccbf4e4773497e334b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3276cda3d08f46609c83cd7afaf47cb1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/519d18c65f684e818dfeca2ee0d2535c] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=46.9 K 2024-11-07T12:36:10,249 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 66728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:10,249 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:10,249 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,249 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b55e24593c4d47bf86ff8dd3df51b63b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730982968043 2024-11-07T12:36:10,249 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3c4ef4e42cae41538f9ee1048159479f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/0ff3c41962cb4ad387a6cd00e427dfdb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55cae0d8207546609bc3899ba0ad5cb5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3298618afc5849bb8ffbeac075f76961] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=65.2 K 2024-11-07T12:36:10,250 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b9b8a15a556442ccbf4e4773497e334b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730982968226 2024-11-07T12:36:10,251 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c4ef4e42cae41538f9ee1048159479f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730982968043 2024-11-07T12:36:10,252 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff3c41962cb4ad387a6cd00e427dfdb, keycount=400, bloomtype=ROW, size=23.1 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730982968213 2024-11-07T12:36:10,253 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55cae0d8207546609bc3899ba0ad5cb5, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982968933 2024-11-07T12:36:10,254 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3298618afc5849bb8ffbeac075f76961, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730982969426 2024-11-07T12:36:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:10,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:36:10,255 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3276cda3d08f46609c83cd7afaf47cb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982968964 2024-11-07T12:36:10,256 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 519d18c65f684e818dfeca2ee0d2535c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730982969441 2024-11-07T12:36:10,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/cab9b9f6737e4cd0acf1bdd2a0610797 is 50, key is test_row_0/A:col10/1730982970251/Put/seqid=0 2024-11-07T12:36:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:36:10,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741849_1025 (size=16681) 2024-11-07T12:36:10,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/cab9b9f6737e4cd0acf1bdd2a0610797 2024-11-07T12:36:10,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:36:10,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:10,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/d7714f44a37049adb84cd14dc7702f74 is 50, key is test_row_0/B:col10/1730982970251/Put/seqid=0 2024-11-07T12:36:10,345 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#13 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:10,345 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#14 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:10,347 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/a788ca6681ce44fbb622a7cca8e106ab is 50, key is test_row_0/B:col10/1730982969444/Put/seqid=0 2024-11-07T12:36:10,347 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/87d6e4ab15c94c86837001751e20b112 is 50, key is test_row_0/A:col10/1730982969444/Put/seqid=0 2024-11-07T12:36:10,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983030369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983030377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983030378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983030380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983030381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741851_1027 (size=12139) 2024-11-07T12:36:10,422 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/a788ca6681ce44fbb622a7cca8e106ab as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/a788ca6681ce44fbb622a7cca8e106ab 2024-11-07T12:36:10,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741850_1026 (size=12001) 2024-11-07T12:36:10,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741852_1028 (size=12139) 2024-11-07T12:36:10,453 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into a788ca6681ce44fbb622a7cca8e106ab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:10,453 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:10,458 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=12, startTime=1730982970239; duration=0sec 2024-11-07T12:36:10,459 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:10,459 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:10,459 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:10,465 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:10,466 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:10,466 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,467 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/0912bd9049db4378aa488e85ce6e79ba, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c1201e51bfb644e2bb8faad993eddcca, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/8946d2ee109349a0986ba2f15b6d3c11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/fcf5c163b42541a5ab2dc11526f7b423] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=46.9 K 2024-11-07T12:36:10,466 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/87d6e4ab15c94c86837001751e20b112 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/87d6e4ab15c94c86837001751e20b112 2024-11-07T12:36:10,469 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 0912bd9049db4378aa488e85ce6e79ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730982968043 2024-11-07T12:36:10,470 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c1201e51bfb644e2bb8faad993eddcca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730982968226 2024-11-07T12:36:10,471 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8946d2ee109349a0986ba2f15b6d3c11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982968964 2024-11-07T12:36:10,472 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting fcf5c163b42541a5ab2dc11526f7b423, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730982969441 2024-11-07T12:36:10,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:36:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,486 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 87d6e4ab15c94c86837001751e20b112(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:10,486 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:10,486 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=12, startTime=1730982970229; duration=0sec 2024-11-07T12:36:10,486 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:10,486 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:36:10,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983030488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983030498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983030498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,516 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#16 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:10,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983030507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,519 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/36954efc2cd54b22a764f6be210c7221 is 50, key is test_row_0/C:col10/1730982969444/Put/seqid=0 2024-11-07T12:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983030507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741853_1029 (size=12139) 2024-11-07T12:36:10,577 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/36954efc2cd54b22a764f6be210c7221 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/36954efc2cd54b22a764f6be210c7221 2024-11-07T12:36:10,589 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into 36954efc2cd54b22a764f6be210c7221(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:10,589 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:10,589 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=12, startTime=1730982970239; duration=0sec 2024-11-07T12:36:10,589 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:10,589 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:10,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:36:10,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:10,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983030698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983030709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983030708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983030728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983030738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:10,780 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:36:10,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:36:10,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:10,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:36:10,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:10,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/d7714f44a37049adb84cd14dc7702f74 2024-11-07T12:36:10,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/817abd67df864d06af3f0a2f77b3df37 is 50, key is test_row_0/C:col10/1730982970251/Put/seqid=0 2024-11-07T12:36:10,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741854_1030 (size=12001) 2024-11-07T12:36:10,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/817abd67df864d06af3f0a2f77b3df37 2024-11-07T12:36:10,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/cab9b9f6737e4cd0acf1bdd2a0610797 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/cab9b9f6737e4cd0acf1bdd2a0610797 2024-11-07T12:36:10,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/cab9b9f6737e4cd0acf1bdd2a0610797, entries=250, sequenceid=91, filesize=16.3 K 2024-11-07T12:36:10,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/d7714f44a37049adb84cd14dc7702f74 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d7714f44a37049adb84cd14dc7702f74 2024-11-07T12:36:10,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d7714f44a37049adb84cd14dc7702f74, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T12:36:10,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/817abd67df864d06af3f0a2f77b3df37 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/817abd67df864d06af3f0a2f77b3df37 2024-11-07T12:36:10,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/817abd67df864d06af3f0a2f77b3df37, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T12:36:10,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for f7b6889d440fde956272a8aab00142f0 in 697ms, sequenceid=91, compaction requested=false 2024-11-07T12:36:10,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:10,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:36:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:10,954 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:36:10,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/382af1242a7c4fd2a6efe7cfae7ad475 is 50, key is test_row_0/A:col10/1730982970377/Put/seqid=0 2024-11-07T12:36:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:11,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:11,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741855_1031 (size=12001) 2024-11-07T12:36:11,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983031051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983031051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983031055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983031057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983031059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983031167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983031167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983031168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983031178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:36:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983031378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983031378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983031378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983031398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,422 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/382af1242a7c4fd2a6efe7cfae7ad475 2024-11-07T12:36:11,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/7e2eca51c57845a5bdaaccd9772a734d is 50, key is test_row_0/B:col10/1730982970377/Put/seqid=0 2024-11-07T12:36:11,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741856_1032 (size=12001) 2024-11-07T12:36:11,523 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T12:36:11,524 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T12:36:11,526 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:36:11,527 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T12:36:11,527 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T12:36:11,527 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T12:36:11,529 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T12:36:11,529 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T12:36:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983031579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983031688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983031688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983031699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983031708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:11,877 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/7e2eca51c57845a5bdaaccd9772a734d 2024-11-07T12:36:11,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/cccd20d8476243939d98b472e2146e1f is 50, key is test_row_0/C:col10/1730982970377/Put/seqid=0 2024-11-07T12:36:11,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741857_1033 (size=12001) 2024-11-07T12:36:11,991 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/cccd20d8476243939d98b472e2146e1f 2024-11-07T12:36:12,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/382af1242a7c4fd2a6efe7cfae7ad475 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/382af1242a7c4fd2a6efe7cfae7ad475 2024-11-07T12:36:12,016 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/382af1242a7c4fd2a6efe7cfae7ad475, entries=150, sequenceid=119, filesize=11.7 K 2024-11-07T12:36:12,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/7e2eca51c57845a5bdaaccd9772a734d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7e2eca51c57845a5bdaaccd9772a734d 2024-11-07T12:36:12,031 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7e2eca51c57845a5bdaaccd9772a734d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-07T12:36:12,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/cccd20d8476243939d98b472e2146e1f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/cccd20d8476243939d98b472e2146e1f 2024-11-07T12:36:12,046 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/cccd20d8476243939d98b472e2146e1f, entries=150, sequenceid=119, filesize=11.7 K 2024-11-07T12:36:12,050 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for f7b6889d440fde956272a8aab00142f0 in 1097ms, sequenceid=119, compaction requested=true 2024-11-07T12:36:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-07T12:36:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-07T12:36:12,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T12:36:12,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-07T12:36:12,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8950 sec 2024-11-07T12:36:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:12,206 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:12,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:36:12,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5f6a00582619480889071aa40f09cd18 is 50, key is test_row_0/A:col10/1730982972199/Put/seqid=0 2024-11-07T12:36:12,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741858_1034 (size=12101) 2024-11-07T12:36:12,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5f6a00582619480889071aa40f09cd18 2024-11-07T12:36:12,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/4cfdfc00db1c4676b0f213cf7fd7d2de is 50, key is test_row_0/B:col10/1730982972199/Put/seqid=0 2024-11-07T12:36:12,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741859_1035 (size=12101) 2024-11-07T12:36:12,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/4cfdfc00db1c4676b0f213cf7fd7d2de 2024-11-07T12:36:12,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983032297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:36:12,318 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:12,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/358ef9e9e59e49ae89a92ecb4e80bfcb is 50, key is test_row_0/C:col10/1730982972199/Put/seqid=0 2024-11-07T12:36:12,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:12,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983032315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983032312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983032353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-07T12:36:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:36:12,368 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:12,370 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:12,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:12,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741860_1036 (size=12101) 2024-11-07T12:36:12,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983032428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983032468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983032468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983032468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:36:12,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:12,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:12,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983032597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,604 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in f7b6889d440fde956272a8aab00142f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:36:12,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983032638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:12,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:12,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983032677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983032677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983032678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:36:12,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/358ef9e9e59e49ae89a92ecb4e80bfcb 2024-11-07T12:36:12,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5f6a00582619480889071aa40f09cd18 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5f6a00582619480889071aa40f09cd18 2024-11-07T12:36:12,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5f6a00582619480889071aa40f09cd18, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T12:36:12,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/4cfdfc00db1c4676b0f213cf7fd7d2de as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4cfdfc00db1c4676b0f213cf7fd7d2de 2024-11-07T12:36:12,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4cfdfc00db1c4676b0f213cf7fd7d2de, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T12:36:12,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:12,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:12,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/358ef9e9e59e49ae89a92ecb4e80bfcb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/358ef9e9e59e49ae89a92ecb4e80bfcb 2024-11-07T12:36:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/358ef9e9e59e49ae89a92ecb4e80bfcb, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T12:36:12,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7b6889d440fde956272a8aab00142f0 in 652ms, sequenceid=132, compaction requested=true 2024-11-07T12:36:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:12,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:12,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:12,859 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:12,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:12,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:12,859 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:12,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:12,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:12,862 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:12,862 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:12,862 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,862 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/a788ca6681ce44fbb622a7cca8e106ab, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d7714f44a37049adb84cd14dc7702f74, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7e2eca51c57845a5bdaaccd9772a734d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4cfdfc00db1c4676b0f213cf7fd7d2de] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=47.1 K 2024-11-07T12:36:12,863 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52922 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:12,863 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a788ca6681ce44fbb622a7cca8e106ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730982969441 2024-11-07T12:36:12,863 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:12,863 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,864 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/87d6e4ab15c94c86837001751e20b112, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/cab9b9f6737e4cd0acf1bdd2a0610797, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/382af1242a7c4fd2a6efe7cfae7ad475, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5f6a00582619480889071aa40f09cd18] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=51.7 K 2024-11-07T12:36:12,864 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d6e4ab15c94c86837001751e20b112, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730982969441 2024-11-07T12:36:12,865 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d7714f44a37049adb84cd14dc7702f74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982969586 2024-11-07T12:36:12,865 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting cab9b9f6737e4cd0acf1bdd2a0610797, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982969586 2024-11-07T12:36:12,866 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 382af1242a7c4fd2a6efe7cfae7ad475, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982970350 2024-11-07T12:36:12,866 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2eca51c57845a5bdaaccd9772a734d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982970350 2024-11-07T12:36:12,867 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6a00582619480889071aa40f09cd18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730982971032 2024-11-07T12:36:12,868 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cfdfc00db1c4676b0f213cf7fd7d2de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730982971032 2024-11-07T12:36:12,896 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:12,897 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/7f2b28a39d794e98a7c24fe65e0d8aad is 50, key is test_row_0/B:col10/1730982972199/Put/seqid=0 2024-11-07T12:36:12,908 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:12,909 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/294d224ab402482f9534b4c2c722046a is 50, key is test_row_0/A:col10/1730982972199/Put/seqid=0 2024-11-07T12:36:12,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741861_1037 (size=12375) 2024-11-07T12:36:12,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741862_1038 (size=12375) 2024-11-07T12:36:12,963 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/7f2b28a39d794e98a7c24fe65e0d8aad as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7f2b28a39d794e98a7c24fe65e0d8aad 2024-11-07T12:36:12,970 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:12,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:36:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:12,975 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into 7f2b28a39d794e98a7c24fe65e0d8aad(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:12,976 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:12,976 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=12, startTime=1730982972859; duration=0sec 2024-11-07T12:36:12,976 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:12,976 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:12,976 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:12,980 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:12,980 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:12,980 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,981 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/36954efc2cd54b22a764f6be210c7221, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/817abd67df864d06af3f0a2f77b3df37, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/cccd20d8476243939d98b472e2146e1f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/358ef9e9e59e49ae89a92ecb4e80bfcb] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=47.1 K 2024-11-07T12:36:12,988 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 36954efc2cd54b22a764f6be210c7221, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730982969441 2024-11-07T12:36:12,989 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 817abd67df864d06af3f0a2f77b3df37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982969586 2024-11-07T12:36:12,990 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting cccd20d8476243939d98b472e2146e1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982970350 2024-11-07T12:36:12,990 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 358ef9e9e59e49ae89a92ecb4e80bfcb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730982971032 2024-11-07T12:36:12,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:12,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:12,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/1c04a1807664422ca9292fa3080367f2 is 50, key is test_row_0/A:col10/1730982972279/Put/seqid=0 2024-11-07T12:36:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:36:13,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983033010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,017 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:13,018 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b901fdf6cbd5423b8234ccc674b50702 is 50, key is test_row_0/C:col10/1730982972199/Put/seqid=0 2024-11-07T12:36:13,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983033012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983033016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983033016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741863_1039 (size=12151) 2024-11-07T12:36:13,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/1c04a1807664422ca9292fa3080367f2 2024-11-07T12:36:13,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741864_1040 (size=12375) 2024-11-07T12:36:13,094 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b901fdf6cbd5423b8234ccc674b50702 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b901fdf6cbd5423b8234ccc674b50702 2024-11-07T12:36:13,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2ad7e5dba7ab42ff8a2bb64af5d72c14 is 50, key is test_row_0/B:col10/1730982972279/Put/seqid=0 2024-11-07T12:36:13,115 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into b901fdf6cbd5423b8234ccc674b50702(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:13,115 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:13,115 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=12, startTime=1730982972859; duration=0sec 2024-11-07T12:36:13,115 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:13,115 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:13,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983033128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983033128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741865_1041 (size=12151) 2024-11-07T12:36:13,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983033138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983033137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:13,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:13,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983033347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983033348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983033348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983033348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,371 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/294d224ab402482f9534b4c2c722046a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/294d224ab402482f9534b4c2c722046a 2024-11-07T12:36:13,382 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 294d224ab402482f9534b4c2c722046a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:13,382 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:13,382 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=12, startTime=1730982972858; duration=0sec 2024-11-07T12:36:13,382 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:13,382 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:13,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:36:13,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2ad7e5dba7ab42ff8a2bb64af5d72c14 2024-11-07T12:36:13,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/1914fcfd9118477a9d5566873da73763 is 50, key is test_row_0/C:col10/1730982972279/Put/seqid=0 2024-11-07T12:36:13,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741866_1042 (size=12151) 2024-11-07T12:36:13,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:13,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983033658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983033658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983033658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983033659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:13,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:13,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:13,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:13,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:13,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:13,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/1914fcfd9118477a9d5566873da73763 2024-11-07T12:36:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/1c04a1807664422ca9292fa3080367f2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1c04a1807664422ca9292fa3080367f2 2024-11-07T12:36:14,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1c04a1807664422ca9292fa3080367f2, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:36:14,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2ad7e5dba7ab42ff8a2bb64af5d72c14 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2ad7e5dba7ab42ff8a2bb64af5d72c14 2024-11-07T12:36:14,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2ad7e5dba7ab42ff8a2bb64af5d72c14, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:36:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/1914fcfd9118477a9d5566873da73763 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1914fcfd9118477a9d5566873da73763 2024-11-07T12:36:14,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1914fcfd9118477a9d5566873da73763, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:36:14,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f7b6889d440fde956272a8aab00142f0 in 1093ms, sequenceid=156, compaction requested=false 2024-11-07T12:36:14,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:14,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:36:14,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,081 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:36:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/c04e77e3f55c4be3b01f3e651dcfa7c9 is 50, key is test_row_0/A:col10/1730982973013/Put/seqid=0 2024-11-07T12:36:14,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741867_1043 (size=12151) 2024-11-07T12:36:14,148 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/c04e77e3f55c4be3b01f3e651dcfa7c9 2024-11-07T12:36:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/d32051386d154f3c9f3367e06a7a8ffa is 50, key is test_row_0/B:col10/1730982973013/Put/seqid=0 2024-11-07T12:36:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:14,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:14,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741868_1044 (size=12151) 2024-11-07T12:36:14,195 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/d32051386d154f3c9f3367e06a7a8ffa 2024-11-07T12:36:14,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/da9a6bd65ffb49b3b957fd2623de8dcd is 50, key is test_row_0/C:col10/1730982973013/Put/seqid=0 2024-11-07T12:36:14,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741869_1045 (size=12151) 2024-11-07T12:36:14,248 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/da9a6bd65ffb49b3b957fd2623de8dcd 2024-11-07T12:36:14,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/c04e77e3f55c4be3b01f3e651dcfa7c9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c04e77e3f55c4be3b01f3e651dcfa7c9 2024-11-07T12:36:14,273 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c04e77e3f55c4be3b01f3e651dcfa7c9, entries=150, sequenceid=171, filesize=11.9 K 2024-11-07T12:36:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/d32051386d154f3c9f3367e06a7a8ffa as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d32051386d154f3c9f3367e06a7a8ffa 2024-11-07T12:36:14,285 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d32051386d154f3c9f3367e06a7a8ffa, entries=150, sequenceid=171, filesize=11.9 K 2024-11-07T12:36:14,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/da9a6bd65ffb49b3b957fd2623de8dcd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/da9a6bd65ffb49b3b957fd2623de8dcd 2024-11-07T12:36:14,300 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/da9a6bd65ffb49b3b957fd2623de8dcd, entries=150, sequenceid=171, filesize=11.9 K 2024-11-07T12:36:14,306 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=134.18 KB/137400 for f7b6889d440fde956272a8aab00142f0 in 225ms, sequenceid=171, compaction requested=true 2024-11-07T12:36:14,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:14,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-07T12:36:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-07T12:36:14,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-07T12:36:14,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9380 sec 2024-11-07T12:36:14,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9540 sec 2024-11-07T12:36:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:14,318 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:14,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:36:14,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/7f30af4d043241239ebb429cf73980c4 is 50, key is test_row_0/A:col10/1730982974307/Put/seqid=0 2024-11-07T12:36:14,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983034347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983034349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983034354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983034352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741870_1046 (size=12151) 2024-11-07T12:36:14,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/7f30af4d043241239ebb429cf73980c4 2024-11-07T12:36:14,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/c1db7cae42124d8faeb016d85cfc9d12 is 50, key is test_row_0/B:col10/1730982974307/Put/seqid=0 2024-11-07T12:36:14,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741871_1047 (size=12151) 2024-11-07T12:36:14,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/c1db7cae42124d8faeb016d85cfc9d12 2024-11-07T12:36:14,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983034457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983034469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983034468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983034469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/1f8301357180432ba28bfb48435cbdc5 is 50, key is test_row_0/C:col10/1730982974307/Put/seqid=0 2024-11-07T12:36:14,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741872_1048 (size=12151) 2024-11-07T12:36:14,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/1f8301357180432ba28bfb48435cbdc5 2024-11-07T12:36:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:36:14,518 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:14,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-07T12:36:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:36:14,525 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:14,528 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:14,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:14,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/7f30af4d043241239ebb429cf73980c4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7f30af4d043241239ebb429cf73980c4 2024-11-07T12:36:14,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7f30af4d043241239ebb429cf73980c4, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T12:36:14,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/c1db7cae42124d8faeb016d85cfc9d12 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/c1db7cae42124d8faeb016d85cfc9d12 2024-11-07T12:36:14,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/c1db7cae42124d8faeb016d85cfc9d12, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T12:36:14,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/1f8301357180432ba28bfb48435cbdc5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1f8301357180432ba28bfb48435cbdc5 2024-11-07T12:36:14,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1f8301357180432ba28bfb48435cbdc5, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T12:36:14,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f7b6889d440fde956272a8aab00142f0 in 260ms, sequenceid=196, compaction requested=true 2024-11-07T12:36:14,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:14,579 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:36:14,579 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:14,581 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:14,581 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:14,581 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,582 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/294d224ab402482f9534b4c2c722046a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1c04a1807664422ca9292fa3080367f2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c04e77e3f55c4be3b01f3e651dcfa7c9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7f30af4d043241239ebb429cf73980c4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=47.7 K 2024-11-07T12:36:14,582 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:14,582 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:14,582 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 294d224ab402482f9534b4c2c722046a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730982971032 2024-11-07T12:36:14,582 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,582 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7f2b28a39d794e98a7c24fe65e0d8aad, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2ad7e5dba7ab42ff8a2bb64af5d72c14, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d32051386d154f3c9f3367e06a7a8ffa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/c1db7cae42124d8faeb016d85cfc9d12] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=47.7 K 2024-11-07T12:36:14,583 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c04a1807664422ca9292fa3080367f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730982972279 2024-11-07T12:36:14,583 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2b28a39d794e98a7c24fe65e0d8aad, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730982971032 2024-11-07T12:36:14,584 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad7e5dba7ab42ff8a2bb64af5d72c14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730982972279 2024-11-07T12:36:14,584 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c04e77e3f55c4be3b01f3e651dcfa7c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982973007 2024-11-07T12:36:14,584 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32051386d154f3c9f3367e06a7a8ffa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982973007 2024-11-07T12:36:14,584 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f30af4d043241239ebb429cf73980c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982974294 2024-11-07T12:36:14,585 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1db7cae42124d8faeb016d85cfc9d12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982974294 2024-11-07T12:36:14,611 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:14,612 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/a227cf0ef0aa4c4da9069834353862ac is 50, key is test_row_0/A:col10/1730982974307/Put/seqid=0 2024-11-07T12:36:14,625 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#37 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:14,626 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/4692c935e13143fe8e010de34f4ef595 is 50, key is test_row_0/B:col10/1730982974307/Put/seqid=0 2024-11-07T12:36:14,626 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:14,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:36:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:36:14,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/91645f745abd4efbbf3036e4ef878e97 is 50, key is test_row_0/A:col10/1730982974347/Put/seqid=0 2024-11-07T12:36:14,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741873_1049 (size=12561) 2024-11-07T12:36:14,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:14,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:14,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741874_1050 (size=12561) 2024-11-07T12:36:14,705 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/4692c935e13143fe8e010de34f4ef595 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4692c935e13143fe8e010de34f4ef595 2024-11-07T12:36:14,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741875_1051 (size=12147) 2024-11-07T12:36:14,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/91645f745abd4efbbf3036e4ef878e97 2024-11-07T12:36:14,717 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into 4692c935e13143fe8e010de34f4ef595(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:14,717 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:14,717 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=12, startTime=1730982974579; duration=0sec 2024-11-07T12:36:14,718 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:14,718 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:14,718 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:14,722 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:14,722 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:14,722 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,722 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b901fdf6cbd5423b8234ccc674b50702, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1914fcfd9118477a9d5566873da73763, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/da9a6bd65ffb49b3b957fd2623de8dcd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1f8301357180432ba28bfb48435cbdc5] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=47.7 K 2024-11-07T12:36:14,723 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b901fdf6cbd5423b8234ccc674b50702, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730982971032 2024-11-07T12:36:14,724 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1914fcfd9118477a9d5566873da73763, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730982972279 2024-11-07T12:36:14,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,725 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting da9a6bd65ffb49b3b957fd2623de8dcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982973007 2024-11-07T12:36:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983034720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/e7818644c563491dbb7adddbfb61171b is 50, key is test_row_0/B:col10/1730982974347/Put/seqid=0 2024-11-07T12:36:14,726 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f8301357180432ba28bfb48435cbdc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982974294 2024-11-07T12:36:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741876_1052 (size=9757) 2024-11-07T12:36:14,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983034721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,741 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:14,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/e7818644c563491dbb7adddbfb61171b 2024-11-07T12:36:14,742 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c83ea9e1f0b5466f87e2d5d50f489bb9 is 50, key is test_row_0/C:col10/1730982974307/Put/seqid=0 2024-11-07T12:36:14,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983034724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983034725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983034725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741877_1053 (size=12561) 2024-11-07T12:36:14,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c is 50, key is test_row_0/C:col10/1730982974347/Put/seqid=0 2024-11-07T12:36:14,776 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c83ea9e1f0b5466f87e2d5d50f489bb9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c83ea9e1f0b5466f87e2d5d50f489bb9 2024-11-07T12:36:14,786 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into c83ea9e1f0b5466f87e2d5d50f489bb9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:14,786 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:14,786 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=12, startTime=1730982974579; duration=0sec 2024-11-07T12:36:14,786 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:14,786 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:14,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741878_1054 (size=9757) 2024-11-07T12:36:14,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c 2024-11-07T12:36:14,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/91645f745abd4efbbf3036e4ef878e97 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/91645f745abd4efbbf3036e4ef878e97 2024-11-07T12:36:14,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/91645f745abd4efbbf3036e4ef878e97, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T12:36:14,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/e7818644c563491dbb7adddbfb61171b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/e7818644c563491dbb7adddbfb61171b 2024-11-07T12:36:14,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/e7818644c563491dbb7adddbfb61171b, entries=100, sequenceid=208, filesize=9.5 K 2024-11-07T12:36:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:36:14,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c 2024-11-07T12:36:14,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:14,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:14,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983034838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c, entries=100, sequenceid=208, filesize=9.5 K 2024-11-07T12:36:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f7b6889d440fde956272a8aab00142f0 in 226ms, sequenceid=208, compaction requested=false 2024-11-07T12:36:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:14,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:14,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:36:14,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/d9736de1601a4e39bf864ecd3501f81d is 50, key is test_row_0/A:col10/1730982974856/Put/seqid=0 2024-11-07T12:36:14,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983034880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983034880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983034885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983034888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:14,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741879_1055 (size=14541) 2024-11-07T12:36:14,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/d9736de1601a4e39bf864ecd3501f81d 2024-11-07T12:36:14,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/935aa85b9ee74f4fb91c37f35ad49a61 is 50, key is test_row_0/B:col10/1730982974856/Put/seqid=0 2024-11-07T12:36:14,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741880_1056 (size=12151) 2024-11-07T12:36:14,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/935aa85b9ee74f4fb91c37f35ad49a61 2024-11-07T12:36:14,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b44d77e555c149ef8927af71943bb53b is 50, key is test_row_0/C:col10/1730982974856/Put/seqid=0 2024-11-07T12:36:14,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:14,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983034997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983034998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983035007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983035008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741881_1057 (size=12151) 2024-11-07T12:36:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983035057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,084 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/a227cf0ef0aa4c4da9069834353862ac as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/a227cf0ef0aa4c4da9069834353862ac 2024-11-07T12:36:15,094 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into a227cf0ef0aa4c4da9069834353862ac(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:15,094 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:15,094 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=12, startTime=1730982974578; duration=0sec 2024-11-07T12:36:15,094 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:15,094 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:36:15,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983035207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983035217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983035217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983035219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:15,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:15,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983035368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b44d77e555c149ef8927af71943bb53b 2024-11-07T12:36:15,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/d9736de1601a4e39bf864ecd3501f81d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/d9736de1601a4e39bf864ecd3501f81d 2024-11-07T12:36:15,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/d9736de1601a4e39bf864ecd3501f81d, entries=200, sequenceid=235, filesize=14.2 K 2024-11-07T12:36:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/935aa85b9ee74f4fb91c37f35ad49a61 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/935aa85b9ee74f4fb91c37f35ad49a61 2024-11-07T12:36:15,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/935aa85b9ee74f4fb91c37f35ad49a61, entries=150, sequenceid=235, filesize=11.9 K 2024-11-07T12:36:15,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b44d77e555c149ef8927af71943bb53b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b44d77e555c149ef8927af71943bb53b 2024-11-07T12:36:15,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b44d77e555c149ef8927af71943bb53b, entries=150, sequenceid=235, filesize=11.9 K 2024-11-07T12:36:15,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f7b6889d440fde956272a8aab00142f0 in 625ms, sequenceid=235, compaction requested=true 2024-11-07T12:36:15,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:15,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:15,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:15,484 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:15,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:15,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:15,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:15,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:36:15,485 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:15,486 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:15,486 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:15,486 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,486 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4692c935e13143fe8e010de34f4ef595, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/e7818644c563491dbb7adddbfb61171b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/935aa85b9ee74f4fb91c37f35ad49a61] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=33.7 K 2024-11-07T12:36:15,487 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:15,487 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:15,487 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,487 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/a227cf0ef0aa4c4da9069834353862ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/91645f745abd4efbbf3036e4ef878e97, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/d9736de1601a4e39bf864ecd3501f81d] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=38.3 K 2024-11-07T12:36:15,487 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4692c935e13143fe8e010de34f4ef595, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982974294 2024-11-07T12:36:15,488 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a227cf0ef0aa4c4da9069834353862ac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982974294 2024-11-07T12:36:15,488 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e7818644c563491dbb7adddbfb61171b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982974347 2024-11-07T12:36:15,489 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91645f745abd4efbbf3036e4ef878e97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982974347 2024-11-07T12:36:15,489 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 935aa85b9ee74f4fb91c37f35ad49a61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730982974699 2024-11-07T12:36:15,489 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9736de1601a4e39bf864ecd3501f81d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730982974699 2024-11-07T12:36:15,507 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:15,508 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/d611acd99aaa4639b661dbb63756f9fa is 50, key is test_row_0/B:col10/1730982974856/Put/seqid=0 2024-11-07T12:36:15,519 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:15,520 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3f60a04baccf4b489cb1cf6a9cab291d is 50, key is test_row_0/A:col10/1730982974856/Put/seqid=0 2024-11-07T12:36:15,530 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:15,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:36:15,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741882_1058 (size=12663) 2024-11-07T12:36:15,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/9720cf4529304a86abcef3b29e430613 is 50, key is test_row_0/A:col10/1730982974875/Put/seqid=0 2024-11-07T12:36:15,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983035587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983035591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741883_1059 (size=12663) 2024-11-07T12:36:15,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983035595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983035597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:15,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741884_1060 (size=12151) 2024-11-07T12:36:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:36:15,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983035708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983035708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983035719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983035719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:15,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:15,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983035878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983035927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983035928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:15,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983035938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983035938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:15,964 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/d611acd99aaa4639b661dbb63756f9fa as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d611acd99aaa4639b661dbb63756f9fa 2024-11-07T12:36:15,974 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into d611acd99aaa4639b661dbb63756f9fa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:15,974 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:15,974 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=13, startTime=1730982975484; duration=0sec 2024-11-07T12:36:15,974 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:15,974 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:15,974 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:15,976 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:15,976 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:15,976 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:15,976 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c83ea9e1f0b5466f87e2d5d50f489bb9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b44d77e555c149ef8927af71943bb53b] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=33.7 K 2024-11-07T12:36:15,977 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c83ea9e1f0b5466f87e2d5d50f489bb9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982974294 2024-11-07T12:36:15,978 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b2cfa5cd9d8242c0bdf88dbcd1885f9c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982974347 2024-11-07T12:36:15,979 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b44d77e555c149ef8927af71943bb53b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730982974699 2024-11-07T12:36:15,992 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:15,992 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/62fea264893d492581a21cdc1398ec39 is 50, key is test_row_0/C:col10/1730982974856/Put/seqid=0 2024-11-07T12:36:15,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741885_1061 (size=12663) 2024-11-07T12:36:16,016 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3f60a04baccf4b489cb1cf6a9cab291d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3f60a04baccf4b489cb1cf6a9cab291d 2024-11-07T12:36:16,025 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 3f60a04baccf4b489cb1cf6a9cab291d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:16,025 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:16,025 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=13, startTime=1730982975484; duration=0sec 2024-11-07T12:36:16,025 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:16,025 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:16,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/9720cf4529304a86abcef3b29e430613 2024-11-07T12:36:16,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9ba9bc8eb93247a6bd05dda91003dc1a is 50, key is test_row_0/B:col10/1730982974875/Put/seqid=0 2024-11-07T12:36:16,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741886_1062 (size=12151) 2024-11-07T12:36:16,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:16,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:16,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9ba9bc8eb93247a6bd05dda91003dc1a 2024-11-07T12:36:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:16,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:16,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/e700a0c1865a43ce95a8dc57ce3ce218 is 50, key is test_row_0/C:col10/1730982974875/Put/seqid=0 2024-11-07T12:36:16,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741887_1063 (size=12151) 2024-11-07T12:36:16,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/e700a0c1865a43ce95a8dc57ce3ce218 2024-11-07T12:36:16,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/9720cf4529304a86abcef3b29e430613 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/9720cf4529304a86abcef3b29e430613 2024-11-07T12:36:16,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/9720cf4529304a86abcef3b29e430613, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T12:36:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9ba9bc8eb93247a6bd05dda91003dc1a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9ba9bc8eb93247a6bd05dda91003dc1a 2024-11-07T12:36:16,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9ba9bc8eb93247a6bd05dda91003dc1a, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T12:36:16,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/e700a0c1865a43ce95a8dc57ce3ce218 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e700a0c1865a43ce95a8dc57ce3ce218 2024-11-07T12:36:16,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e700a0c1865a43ce95a8dc57ce3ce218, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T12:36:16,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f7b6889d440fde956272a8aab00142f0 in 663ms, sequenceid=248, compaction requested=false 2024-11-07T12:36:16,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:16,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:36:16,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:16,273 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:36:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:16,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/c5e5f816d9164976ac7cd3028c6176c3 is 50, key is test_row_0/A:col10/1730982975584/Put/seqid=0 2024-11-07T12:36:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983036284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983036285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983036288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983036293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741888_1064 (size=12301) 2024-11-07T12:36:16,315 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/c5e5f816d9164976ac7cd3028c6176c3 2024-11-07T12:36:16,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2153b82e98334a7bac222d0b24650b50 is 50, key is test_row_0/B:col10/1730982975584/Put/seqid=0 2024-11-07T12:36:16,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741889_1065 (size=12301) 2024-11-07T12:36:16,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983036398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983036398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983036408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983036408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,420 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/62fea264893d492581a21cdc1398ec39 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/62fea264893d492581a21cdc1398ec39 2024-11-07T12:36:16,431 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into 62fea264893d492581a21cdc1398ec39(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:16,431 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:16,431 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=13, startTime=1730982975484; duration=0sec 2024-11-07T12:36:16,432 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:16,432 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:16,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983036607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983036618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983036618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983036618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:36:16,748 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2153b82e98334a7bac222d0b24650b50 2024-11-07T12:36:16,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/821d161c93b74f1db465809089f46ef2 is 50, key is test_row_0/C:col10/1730982975584/Put/seqid=0 2024-11-07T12:36:16,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741890_1066 (size=12301) 2024-11-07T12:36:16,814 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/821d161c93b74f1db465809089f46ef2 2024-11-07T12:36:16,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/c5e5f816d9164976ac7cd3028c6176c3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c5e5f816d9164976ac7cd3028c6176c3 2024-11-07T12:36:16,832 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c5e5f816d9164976ac7cd3028c6176c3, entries=150, sequenceid=274, filesize=12.0 K 2024-11-07T12:36:16,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2153b82e98334a7bac222d0b24650b50 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2153b82e98334a7bac222d0b24650b50 2024-11-07T12:36:16,841 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2153b82e98334a7bac222d0b24650b50, entries=150, sequenceid=274, filesize=12.0 K 2024-11-07T12:36:16,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/821d161c93b74f1db465809089f46ef2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/821d161c93b74f1db465809089f46ef2 2024-11-07T12:36:16,850 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/821d161c93b74f1db465809089f46ef2, entries=150, sequenceid=274, filesize=12.0 K 2024-11-07T12:36:16,853 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f7b6889d440fde956272a8aab00142f0 in 580ms, sequenceid=274, compaction requested=true 2024-11-07T12:36:16,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:16,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:16,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-07T12:36:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-07T12:36:16,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T12:36:16,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3270 sec 2024-11-07T12:36:16,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3400 sec 2024-11-07T12:36:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:16,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:36:16,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3880a425c53a404b877286f5153209d4 is 50, key is test_row_0/A:col10/1730982976891/Put/seqid=0 2024-11-07T12:36:16,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741891_1067 (size=12301) 2024-11-07T12:36:16,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3880a425c53a404b877286f5153209d4 2024-11-07T12:36:16,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/56a5c2d95ed146a5b22c96a4ba68fcc9 is 50, key is test_row_0/B:col10/1730982976891/Put/seqid=0 2024-11-07T12:36:16,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983036945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983036949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983036951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741892_1068 (size=12301) 2024-11-07T12:36:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/56a5c2d95ed146a5b22c96a4ba68fcc9 2024-11-07T12:36:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983036958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983036963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:16,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/9dcbec8e9f514430aa9b2d639a279dd8 is 50, key is test_row_0/C:col10/1730982976891/Put/seqid=0 2024-11-07T12:36:16,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741893_1069 (size=12301) 2024-11-07T12:36:16,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/9dcbec8e9f514430aa9b2d639a279dd8 2024-11-07T12:36:17,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/3880a425c53a404b877286f5153209d4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3880a425c53a404b877286f5153209d4 2024-11-07T12:36:17,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3880a425c53a404b877286f5153209d4, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T12:36:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/56a5c2d95ed146a5b22c96a4ba68fcc9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/56a5c2d95ed146a5b22c96a4ba68fcc9 2024-11-07T12:36:17,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/56a5c2d95ed146a5b22c96a4ba68fcc9, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T12:36:17,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/9dcbec8e9f514430aa9b2d639a279dd8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9dcbec8e9f514430aa9b2d639a279dd8 2024-11-07T12:36:17,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9dcbec8e9f514430aa9b2d639a279dd8, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T12:36:17,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7b6889d440fde956272a8aab00142f0 in 145ms, sequenceid=288, compaction requested=true 2024-11-07T12:36:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:17,038 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:17,038 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:17,040 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:17,040 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:17,040 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:17,040 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:17,040 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3f60a04baccf4b489cb1cf6a9cab291d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/9720cf4529304a86abcef3b29e430613, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c5e5f816d9164976ac7cd3028c6176c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3880a425c53a404b877286f5153209d4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=48.3 K 2024-11-07T12:36:17,040 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:17,041 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:17,041 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d611acd99aaa4639b661dbb63756f9fa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9ba9bc8eb93247a6bd05dda91003dc1a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2153b82e98334a7bac222d0b24650b50, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/56a5c2d95ed146a5b22c96a4ba68fcc9] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=48.3 K 2024-11-07T12:36:17,041 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f60a04baccf4b489cb1cf6a9cab291d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730982974699 2024-11-07T12:36:17,041 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d611acd99aaa4639b661dbb63756f9fa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730982974699 2024-11-07T12:36:17,042 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9720cf4529304a86abcef3b29e430613, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730982974875 2024-11-07T12:36:17,042 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba9bc8eb93247a6bd05dda91003dc1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730982974875 2024-11-07T12:36:17,042 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e5f816d9164976ac7cd3028c6176c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1730982975584 2024-11-07T12:36:17,042 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2153b82e98334a7bac222d0b24650b50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1730982975584 2024-11-07T12:36:17,043 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3880a425c53a404b877286f5153209d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730982976281 2024-11-07T12:36:17,043 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a5c2d95ed146a5b22c96a4ba68fcc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730982976281 2024-11-07T12:36:17,063 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:17,064 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/f530645b372c4070bbad7d12a3674e6e is 50, key is test_row_0/B:col10/1730982976891/Put/seqid=0 2024-11-07T12:36:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:17,074 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:17,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:36:17,074 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:17,075 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/830b8f022ac7447eae018e64df8a80b2 is 50, key is test_row_0/A:col10/1730982976891/Put/seqid=0 2024-11-07T12:36:17,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/405798dc677e464089978a0dea3ea830 is 50, key is test_row_0/A:col10/1730982977071/Put/seqid=0 2024-11-07T12:36:17,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741894_1070 (size=12949) 2024-11-07T12:36:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741895_1071 (size=12949) 2024-11-07T12:36:17,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741896_1072 (size=12301) 2024-11-07T12:36:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983037089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983037091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983037128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983037129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983037129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983037238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983037238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983037247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983037248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983037248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983037447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983037448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983037458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983037458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983037458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,525 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/f530645b372c4070bbad7d12a3674e6e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/f530645b372c4070bbad7d12a3674e6e 2024-11-07T12:36:17,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/405798dc677e464089978a0dea3ea830 2024-11-07T12:36:17,533 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/830b8f022ac7447eae018e64df8a80b2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/830b8f022ac7447eae018e64df8a80b2 2024-11-07T12:36:17,543 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into f530645b372c4070bbad7d12a3674e6e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:17,543 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:17,543 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=12, startTime=1730982977038; duration=0sec 2024-11-07T12:36:17,543 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:17,543 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:17,543 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:17,544 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 830b8f022ac7447eae018e64df8a80b2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:17,545 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:17,545 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=12, startTime=1730982977038; duration=0sec 2024-11-07T12:36:17,545 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:17,545 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:17,548 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:17,548 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:17,548 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:17,548 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/62fea264893d492581a21cdc1398ec39, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e700a0c1865a43ce95a8dc57ce3ce218, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/821d161c93b74f1db465809089f46ef2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9dcbec8e9f514430aa9b2d639a279dd8] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=48.3 K 2024-11-07T12:36:17,549 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fea264893d492581a21cdc1398ec39, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730982974699 2024-11-07T12:36:17,549 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e700a0c1865a43ce95a8dc57ce3ce218, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730982974875 2024-11-07T12:36:17,550 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 821d161c93b74f1db465809089f46ef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1730982975584 2024-11-07T12:36:17,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3337ce5eb3e145d69c90bc83fca3349b is 50, key is test_row_0/B:col10/1730982977071/Put/seqid=0 2024-11-07T12:36:17,550 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcbec8e9f514430aa9b2d639a279dd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730982976281 2024-11-07T12:36:17,578 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:17,579 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/137e624bc93d4292bfed656c7649caba is 50, key is test_row_0/C:col10/1730982976891/Put/seqid=0 2024-11-07T12:36:17,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741897_1073 (size=12301) 2024-11-07T12:36:17,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3337ce5eb3e145d69c90bc83fca3349b 2024-11-07T12:36:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741898_1074 (size=12949) 2024-11-07T12:36:17,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/16c86a8160e742d6af61de976a8bb552 is 50, key is test_row_0/C:col10/1730982977071/Put/seqid=0 2024-11-07T12:36:17,616 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/137e624bc93d4292bfed656c7649caba as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/137e624bc93d4292bfed656c7649caba 2024-11-07T12:36:17,628 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into 137e624bc93d4292bfed656c7649caba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:17,628 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:17,628 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=12, startTime=1730982977038; duration=0sec 2024-11-07T12:36:17,629 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:17,629 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:17,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741899_1075 (size=12301) 2024-11-07T12:36:17,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/16c86a8160e742d6af61de976a8bb552 2024-11-07T12:36:17,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/405798dc677e464089978a0dea3ea830 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/405798dc677e464089978a0dea3ea830 2024-11-07T12:36:17,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/405798dc677e464089978a0dea3ea830, entries=150, sequenceid=313, filesize=12.0 K 2024-11-07T12:36:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3337ce5eb3e145d69c90bc83fca3349b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3337ce5eb3e145d69c90bc83fca3349b 2024-11-07T12:36:17,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3337ce5eb3e145d69c90bc83fca3349b, entries=150, sequenceid=313, filesize=12.0 K 2024-11-07T12:36:17,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/16c86a8160e742d6af61de976a8bb552 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/16c86a8160e742d6af61de976a8bb552 2024-11-07T12:36:17,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/16c86a8160e742d6af61de976a8bb552, entries=150, sequenceid=313, filesize=12.0 K 2024-11-07T12:36:17,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for f7b6889d440fde956272a8aab00142f0 in 605ms, sequenceid=313, compaction requested=false 2024-11-07T12:36:17,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:17,760 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:17,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:36:17,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/7170b4aed886410aaf19596469751a93 is 50, key is test_row_0/A:col10/1730982977092/Put/seqid=0 2024-11-07T12:36:17,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741900_1076 (size=12301) 2024-11-07T12:36:17,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983037806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983037811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983037812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983037813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983037813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983037917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983037928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983037928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983037928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983037928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983038127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983038137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983038138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983038138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983038140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/7170b4aed886410aaf19596469751a93 2024-11-07T12:36:18,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9fc78477344b4dc19cae9b7cd4717dd7 is 50, key is test_row_0/B:col10/1730982977092/Put/seqid=0 2024-11-07T12:36:18,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741901_1077 (size=12301) 2024-11-07T12:36:18,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9fc78477344b4dc19cae9b7cd4717dd7 2024-11-07T12:36:18,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/a5d730195be543c18d729ba98e9f363f is 50, key is test_row_0/C:col10/1730982977092/Put/seqid=0 2024-11-07T12:36:18,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741902_1078 (size=12301) 2024-11-07T12:36:18,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/a5d730195be543c18d729ba98e9f363f 2024-11-07T12:36:18,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/7170b4aed886410aaf19596469751a93 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7170b4aed886410aaf19596469751a93 2024-11-07T12:36:18,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7170b4aed886410aaf19596469751a93, entries=150, sequenceid=329, filesize=12.0 K 2024-11-07T12:36:18,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9fc78477344b4dc19cae9b7cd4717dd7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9fc78477344b4dc19cae9b7cd4717dd7 2024-11-07T12:36:18,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9fc78477344b4dc19cae9b7cd4717dd7, entries=150, sequenceid=329, filesize=12.0 K 2024-11-07T12:36:18,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/a5d730195be543c18d729ba98e9f363f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/a5d730195be543c18d729ba98e9f363f 2024-11-07T12:36:18,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/a5d730195be543c18d729ba98e9f363f, entries=150, sequenceid=329, filesize=12.0 K 2024-11-07T12:36:18,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for f7b6889d440fde956272a8aab00142f0 in 523ms, sequenceid=329, compaction requested=true 2024-11-07T12:36:18,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:18,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:18,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:18,283 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:18,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:18,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:18,283 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:18,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:18,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:18,285 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:18,285 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:18,285 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:18,285 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:18,285 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:18,285 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:18,285 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/f530645b372c4070bbad7d12a3674e6e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3337ce5eb3e145d69c90bc83fca3349b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9fc78477344b4dc19cae9b7cd4717dd7] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=36.7 K 2024-11-07T12:36:18,285 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/830b8f022ac7447eae018e64df8a80b2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/405798dc677e464089978a0dea3ea830, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7170b4aed886410aaf19596469751a93] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=36.7 K 2024-11-07T12:36:18,286 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f530645b372c4070bbad7d12a3674e6e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730982976281 2024-11-07T12:36:18,286 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3337ce5eb3e145d69c90bc83fca3349b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1730982976953 2024-11-07T12:36:18,287 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 830b8f022ac7447eae018e64df8a80b2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730982976281 2024-11-07T12:36:18,287 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc78477344b4dc19cae9b7cd4717dd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730982977092 2024-11-07T12:36:18,288 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405798dc677e464089978a0dea3ea830, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1730982976953 2024-11-07T12:36:18,292 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7170b4aed886410aaf19596469751a93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730982977092 2024-11-07T12:36:18,310 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:18,311 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:18,311 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/5e6284a86db54d649e31b3377d741d43 is 50, key is test_row_0/B:col10/1730982977092/Put/seqid=0 2024-11-07T12:36:18,312 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/8a5cbbd9c6474e3d9fa902824ba260e4 is 50, key is test_row_0/A:col10/1730982977092/Put/seqid=0 2024-11-07T12:36:18,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741904_1080 (size=13051) 2024-11-07T12:36:18,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741903_1079 (size=13051) 2024-11-07T12:36:18,393 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/5e6284a86db54d649e31b3377d741d43 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/5e6284a86db54d649e31b3377d741d43 2024-11-07T12:36:18,404 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into 5e6284a86db54d649e31b3377d741d43(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:18,404 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:18,405 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=13, startTime=1730982978283; duration=0sec 2024-11-07T12:36:18,405 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:18,405 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:18,405 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:18,407 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:18,407 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:18,407 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:18,407 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/137e624bc93d4292bfed656c7649caba, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/16c86a8160e742d6af61de976a8bb552, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/a5d730195be543c18d729ba98e9f363f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=36.7 K 2024-11-07T12:36:18,408 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 137e624bc93d4292bfed656c7649caba, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730982976281 2024-11-07T12:36:18,409 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c86a8160e742d6af61de976a8bb552, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1730982976953 2024-11-07T12:36:18,409 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a5d730195be543c18d729ba98e9f363f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730982977092 2024-11-07T12:36:18,420 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:18,421 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/99b1ea81fdf0484eaef199daabb2d635 is 50, key is test_row_0/C:col10/1730982977092/Put/seqid=0 2024-11-07T12:36:18,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741905_1081 (size=13051) 2024-11-07T12:36:18,443 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/99b1ea81fdf0484eaef199daabb2d635 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/99b1ea81fdf0484eaef199daabb2d635 2024-11-07T12:36:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:18,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:36:18,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/8ae030583ae641308202da5f33af753a is 50, key is test_row_0/A:col10/1730982978441/Put/seqid=0 2024-11-07T12:36:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:18,460 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into 99b1ea81fdf0484eaef199daabb2d635(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:18,461 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:18,461 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=13, startTime=1730982978283; duration=0sec 2024-11-07T12:36:18,461 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:18,461 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:18,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741906_1082 (size=14741) 2024-11-07T12:36:18,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983038460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983038461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983038463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983038465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983038471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983038578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983038578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983038578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983038578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983038588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:36:18,688 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:18,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-07T12:36:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:36:18,692 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:18,693 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:18,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:18,765 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/8a5cbbd9c6474e3d9fa902824ba260e4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8a5cbbd9c6474e3d9fa902824ba260e4 2024-11-07T12:36:18,783 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 8a5cbbd9c6474e3d9fa902824ba260e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:18,783 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:18,783 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=13, startTime=1730982978283; duration=0sec 2024-11-07T12:36:18,783 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:18,783 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:18,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983038787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983038787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983038787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983038788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:36:18,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983038797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:18,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:36:18,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:18,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:18,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:18,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:18,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:18,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/8ae030583ae641308202da5f33af753a 2024-11-07T12:36:18,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9f2f6dd7202943a8b416268cf452d667 is 50, key is test_row_0/B:col10/1730982978441/Put/seqid=0 2024-11-07T12:36:18,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741907_1083 (size=12301) 2024-11-07T12:36:18,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9f2f6dd7202943a8b416268cf452d667 2024-11-07T12:36:18,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b98783020b304f818071df94165644c7 is 50, key is test_row_0/C:col10/1730982978441/Put/seqid=0 2024-11-07T12:36:18,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741908_1084 (size=12301) 2024-11-07T12:36:18,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b98783020b304f818071df94165644c7 2024-11-07T12:36:18,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/8ae030583ae641308202da5f33af753a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8ae030583ae641308202da5f33af753a 2024-11-07T12:36:18,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8ae030583ae641308202da5f33af753a, entries=200, sequenceid=355, filesize=14.4 K 2024-11-07T12:36:18,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/9f2f6dd7202943a8b416268cf452d667 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9f2f6dd7202943a8b416268cf452d667 2024-11-07T12:36:18,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9f2f6dd7202943a8b416268cf452d667, entries=150, sequenceid=355, filesize=12.0 K 2024-11-07T12:36:18,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/b98783020b304f818071df94165644c7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b98783020b304f818071df94165644c7 2024-11-07T12:36:18,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b98783020b304f818071df94165644c7, entries=150, sequenceid=355, filesize=12.0 K 2024-11-07T12:36:18,955 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-07T12:36:18,955 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-07T12:36:18,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f7b6889d440fde956272a8aab00142f0 in 511ms, sequenceid=355, compaction requested=false 2024-11-07T12:36:18,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:19,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:36:19,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:19,004 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:36:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:36:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5717b01ba0184038b83b12a80e7dadf1 is 50, key is test_row_0/A:col10/1730982978455/Put/seqid=0 2024-11-07T12:36:19,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741909_1085 (size=12301) 2024-11-07T12:36:19,038 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5717b01ba0184038b83b12a80e7dadf1 2024-11-07T12:36:19,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/4b60aff464b54753b59e38648b2b96a0 is 50, key is test_row_0/B:col10/1730982978455/Put/seqid=0 2024-11-07T12:36:19,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741910_1086 (size=12301) 2024-11-07T12:36:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983039130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983039130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983039132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983039131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983039133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983039248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983039248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983039248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983039249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983039249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:36:19,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983039457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983039458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983039458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983039458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983039458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,474 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/4b60aff464b54753b59e38648b2b96a0 2024-11-07T12:36:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c4202498aefd42fe8d75f07918e44f11 is 50, key is test_row_0/C:col10/1730982978455/Put/seqid=0 2024-11-07T12:36:19,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741911_1087 (size=12301) 2024-11-07T12:36:19,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983039767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983039767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983039768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983039777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983039777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:36:19,904 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c4202498aefd42fe8d75f07918e44f11 2024-11-07T12:36:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5717b01ba0184038b83b12a80e7dadf1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5717b01ba0184038b83b12a80e7dadf1 2024-11-07T12:36:19,923 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5717b01ba0184038b83b12a80e7dadf1, entries=150, sequenceid=368, filesize=12.0 K 2024-11-07T12:36:19,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/4b60aff464b54753b59e38648b2b96a0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4b60aff464b54753b59e38648b2b96a0 2024-11-07T12:36:19,939 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4b60aff464b54753b59e38648b2b96a0, entries=150, sequenceid=368, filesize=12.0 K 2024-11-07T12:36:19,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c4202498aefd42fe8d75f07918e44f11 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c4202498aefd42fe8d75f07918e44f11 2024-11-07T12:36:19,947 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c4202498aefd42fe8d75f07918e44f11, entries=150, sequenceid=368, filesize=12.0 K 2024-11-07T12:36:19,948 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f7b6889d440fde956272a8aab00142f0 in 944ms, sequenceid=368, compaction requested=true 2024-11-07T12:36:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-07T12:36:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-07T12:36:19,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T12:36:19,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2580 sec 2024-11-07T12:36:19,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2660 sec 2024-11-07T12:36:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:20,282 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:20,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:36:20,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/001e8ecacea64dcab39b67afc705701f is 50, key is test_row_0/A:col10/1730982979129/Put/seqid=0 2024-11-07T12:36:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983040288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983040294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983040289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983040296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983040296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741912_1088 (size=17181) 2024-11-07T12:36:20,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983040408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983040408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983040408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983040409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983040617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983040617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983040618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983040618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/001e8ecacea64dcab39b67afc705701f 2024-11-07T12:36:20,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/bc070fe1f2a1417e90ad59dbfc1e68c0 is 50, key is test_row_0/B:col10/1730982979129/Put/seqid=0 2024-11-07T12:36:20,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741913_1089 (size=12301) 2024-11-07T12:36:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:36:20,838 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:20,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-07T12:36:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:36:20,842 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:20,843 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:20,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:20,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983040928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983040929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983040929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983040929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:36:20,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:36:20,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:20,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:20,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:20,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:20,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:21,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/bc070fe1f2a1417e90ad59dbfc1e68c0 2024-11-07T12:36:21,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c27ceb12a6e24013a96ccd5f84ebb62f is 50, key is test_row_0/C:col10/1730982979129/Put/seqid=0 2024-11-07T12:36:21,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741914_1090 (size=12301) 2024-11-07T12:36:21,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c27ceb12a6e24013a96ccd5f84ebb62f 2024-11-07T12:36:21,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:36:21,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:21,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:21,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:21,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:21,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/001e8ecacea64dcab39b67afc705701f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/001e8ecacea64dcab39b67afc705701f 2024-11-07T12:36:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:36:21,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/001e8ecacea64dcab39b67afc705701f, entries=250, sequenceid=396, filesize=16.8 K 2024-11-07T12:36:21,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/bc070fe1f2a1417e90ad59dbfc1e68c0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/bc070fe1f2a1417e90ad59dbfc1e68c0 2024-11-07T12:36:21,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/bc070fe1f2a1417e90ad59dbfc1e68c0, entries=150, sequenceid=396, filesize=12.0 K 2024-11-07T12:36:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/c27ceb12a6e24013a96ccd5f84ebb62f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c27ceb12a6e24013a96ccd5f84ebb62f 2024-11-07T12:36:21,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c27ceb12a6e24013a96ccd5f84ebb62f, entries=150, sequenceid=396, filesize=12.0 K 2024-11-07T12:36:21,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for f7b6889d440fde956272a8aab00142f0 in 903ms, sequenceid=396, compaction requested=true 2024-11-07T12:36:21,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:21,186 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:21,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:21,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:21,186 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:21,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:21,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:21,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:21,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:21,190 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:21,190 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:21,190 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:21,190 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8a5cbbd9c6474e3d9fa902824ba260e4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8ae030583ae641308202da5f33af753a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5717b01ba0184038b83b12a80e7dadf1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/001e8ecacea64dcab39b67afc705701f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=55.9 K 2024-11-07T12:36:21,191 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:21,191 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a5cbbd9c6474e3d9fa902824ba260e4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730982977092 2024-11-07T12:36:21,191 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:21,191 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:21,191 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/5e6284a86db54d649e31b3377d741d43, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9f2f6dd7202943a8b416268cf452d667, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4b60aff464b54753b59e38648b2b96a0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/bc070fe1f2a1417e90ad59dbfc1e68c0] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=48.8 K 2024-11-07T12:36:21,191 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae030583ae641308202da5f33af753a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982977808 2024-11-07T12:36:21,192 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e6284a86db54d649e31b3377d741d43, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730982977092 2024-11-07T12:36:21,192 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5717b01ba0184038b83b12a80e7dadf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1730982978455 2024-11-07T12:36:21,192 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2f6dd7202943a8b416268cf452d667, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982977808 2024-11-07T12:36:21,192 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001e8ecacea64dcab39b67afc705701f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730982979129 2024-11-07T12:36:21,193 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b60aff464b54753b59e38648b2b96a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1730982978455 2024-11-07T12:36:21,194 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bc070fe1f2a1417e90ad59dbfc1e68c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730982979129 2024-11-07T12:36:21,220 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:21,221 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/90a79d42755a413a8de0194d209fc8fa is 50, key is test_row_0/A:col10/1730982979129/Put/seqid=0 2024-11-07T12:36:21,226 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#79 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:21,227 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3d7db9bf993f4584be74fbefa6e20733 is 50, key is test_row_0/B:col10/1730982979129/Put/seqid=0 2024-11-07T12:36:21,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741915_1091 (size=13187) 2024-11-07T12:36:21,275 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3d7db9bf993f4584be74fbefa6e20733 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3d7db9bf993f4584be74fbefa6e20733 2024-11-07T12:36:21,285 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into 3d7db9bf993f4584be74fbefa6e20733(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:21,286 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:21,286 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=12, startTime=1730982981186; duration=0sec 2024-11-07T12:36:21,287 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:21,287 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:21,287 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:21,289 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:21,289 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:21,289 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:21,289 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/99b1ea81fdf0484eaef199daabb2d635, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b98783020b304f818071df94165644c7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c4202498aefd42fe8d75f07918e44f11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c27ceb12a6e24013a96ccd5f84ebb62f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=48.8 K 2024-11-07T12:36:21,290 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b1ea81fdf0484eaef199daabb2d635, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730982977092 2024-11-07T12:36:21,292 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b98783020b304f818071df94165644c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982977808 2024-11-07T12:36:21,293 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c4202498aefd42fe8d75f07918e44f11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1730982978455 2024-11-07T12:36:21,293 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c27ceb12a6e24013a96ccd5f84ebb62f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730982979129 2024-11-07T12:36:21,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741916_1092 (size=13187) 2024-11-07T12:36:21,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:36:21,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:21,307 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T12:36:21,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/e9fdbef529f64b3ba1520c9313958011 is 50, key is test_row_0/A:col10/1730982980286/Put/seqid=0 2024-11-07T12:36:21,339 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#81 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:21,345 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/09b6e84f3b8546359abae7d6f0a444aa is 50, key is test_row_0/C:col10/1730982979129/Put/seqid=0 2024-11-07T12:36:21,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741917_1093 (size=12301) 2024-11-07T12:36:21,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741918_1094 (size=13187) 2024-11-07T12:36:21,390 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/09b6e84f3b8546359abae7d6f0a444aa as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/09b6e84f3b8546359abae7d6f0a444aa 2024-11-07T12:36:21,399 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into 09b6e84f3b8546359abae7d6f0a444aa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:21,399 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:21,399 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=12, startTime=1730982981186; duration=0sec 2024-11-07T12:36:21,399 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:21,399 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:21,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983041442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983041444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983041447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983041447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983041448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:36:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983041558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983041558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983041558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983041558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,705 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/90a79d42755a413a8de0194d209fc8fa as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/90a79d42755a413a8de0194d209fc8fa 2024-11-07T12:36:21,713 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 90a79d42755a413a8de0194d209fc8fa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:21,713 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:21,713 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=12, startTime=1730982981186; duration=0sec 2024-11-07T12:36:21,714 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:21,714 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:21,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983041768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983041768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983041768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:21,772 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/e9fdbef529f64b3ba1520c9313958011 2024-11-07T12:36:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983041768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:21,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2083d2fe8acf43bd85f2c5afb62223a7 is 50, key is test_row_0/B:col10/1730982980286/Put/seqid=0 2024-11-07T12:36:21,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741919_1095 (size=12301) 2024-11-07T12:36:21,813 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2083d2fe8acf43bd85f2c5afb62223a7 2024-11-07T12:36:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/6511d5b9983549e697e96b83faff70fd is 50, key is test_row_0/C:col10/1730982980286/Put/seqid=0 2024-11-07T12:36:21,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741920_1096 (size=12301) 2024-11-07T12:36:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:36:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983042077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983042078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983042078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983042078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,231 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/6511d5b9983549e697e96b83faff70fd 2024-11-07T12:36:22,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/e9fdbef529f64b3ba1520c9313958011 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e9fdbef529f64b3ba1520c9313958011 2024-11-07T12:36:22,255 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e9fdbef529f64b3ba1520c9313958011, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:36:22,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/2083d2fe8acf43bd85f2c5afb62223a7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2083d2fe8acf43bd85f2c5afb62223a7 2024-11-07T12:36:22,264 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2083d2fe8acf43bd85f2c5afb62223a7, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:36:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/6511d5b9983549e697e96b83faff70fd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/6511d5b9983549e697e96b83faff70fd 2024-11-07T12:36:22,275 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/6511d5b9983549e697e96b83faff70fd, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:36:22,276 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for f7b6889d440fde956272a8aab00142f0 in 969ms, sequenceid=406, compaction requested=false 2024-11-07T12:36:22,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:22,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:22,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-07T12:36:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-07T12:36:22,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-07T12:36:22,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-07T12:36:22,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4840 sec 2024-11-07T12:36:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:22,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:22,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T12:36:22,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/16202eb01c204dd291cd1d9386c5f005 is 50, key is test_row_0/A:col10/1730982981446/Put/seqid=0 2024-11-07T12:36:22,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983042494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741921_1097 (size=14741) 2024-11-07T12:36:22,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/16202eb01c204dd291cd1d9386c5f005 2024-11-07T12:36:22,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/226ca194a2b34b3181695018ce864168 is 50, key is test_row_0/B:col10/1730982981446/Put/seqid=0 2024-11-07T12:36:22,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741922_1098 (size=12301) 2024-11-07T12:36:22,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983042588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983042588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983042589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983042589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983042607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983042818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:22,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/226ca194a2b34b3181695018ce864168 2024-11-07T12:36:22,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/21e165e0832e4283a79b62bdb912b779 is 50, key is test_row_0/C:col10/1730982981446/Put/seqid=0 2024-11-07T12:36:22,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741923_1099 (size=12301) 2024-11-07T12:36:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:36:22,988 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:22,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-07T12:36:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:36:22,992 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:22,994 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:22,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:36:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983043128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:36:23,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:23,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:23,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:23,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:23,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:36:23,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:36:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:23,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:23,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:23,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/21e165e0832e4283a79b62bdb912b779 2024-11-07T12:36:23,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/16202eb01c204dd291cd1d9386c5f005 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/16202eb01c204dd291cd1d9386c5f005 2024-11-07T12:36:23,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/16202eb01c204dd291cd1d9386c5f005, entries=200, sequenceid=436, filesize=14.4 K 2024-11-07T12:36:23,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/226ca194a2b34b3181695018ce864168 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/226ca194a2b34b3181695018ce864168 2024-11-07T12:36:23,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/226ca194a2b34b3181695018ce864168, entries=150, sequenceid=436, filesize=12.0 K 2024-11-07T12:36:23,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/21e165e0832e4283a79b62bdb912b779 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/21e165e0832e4283a79b62bdb912b779 2024-11-07T12:36:23,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/21e165e0832e4283a79b62bdb912b779, entries=150, sequenceid=436, filesize=12.0 K 2024-11-07T12:36:23,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for f7b6889d440fde956272a8aab00142f0 in 955ms, sequenceid=436, compaction requested=true 2024-11-07T12:36:23,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:23,428 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:36:23,428 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:23,430 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:23,430 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:23,430 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:23,431 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/90a79d42755a413a8de0194d209fc8fa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e9fdbef529f64b3ba1520c9313958011, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/16202eb01c204dd291cd1d9386c5f005] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=39.3 K 2024-11-07T12:36:23,431 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:23,431 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:23,431 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:23,431 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3d7db9bf993f4584be74fbefa6e20733, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2083d2fe8acf43bd85f2c5afb62223a7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/226ca194a2b34b3181695018ce864168] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=36.9 K 2024-11-07T12:36:23,432 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a79d42755a413a8de0194d209fc8fa, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730982979129 2024-11-07T12:36:23,432 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d7db9bf993f4584be74fbefa6e20733, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730982979129 2024-11-07T12:36:23,432 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2083d2fe8acf43bd85f2c5afb62223a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730982980286 2024-11-07T12:36:23,433 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e9fdbef529f64b3ba1520c9313958011, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730982980286 2024-11-07T12:36:23,433 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 226ca194a2b34b3181695018ce864168, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730982981444 2024-11-07T12:36:23,433 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 16202eb01c204dd291cd1d9386c5f005, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730982981442 2024-11-07T12:36:23,463 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:23,464 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/6a3593c7d62b444d83af243bbce849c9 is 50, key is test_row_0/B:col10/1730982981446/Put/seqid=0 2024-11-07T12:36:23,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:36:23,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:23,476 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T12:36:23,479 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:23,479 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/057db0716ae148a697ae24a243ca390b is 50, key is test_row_0/A:col10/1730982981446/Put/seqid=0 2024-11-07T12:36:23,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/62e7d2c8a5a949748be567e72c0dabbf is 50, key is test_row_0/A:col10/1730982982480/Put/seqid=0 2024-11-07T12:36:23,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741924_1100 (size=13289) 2024-11-07T12:36:23,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741925_1101 (size=13289) 2024-11-07T12:36:23,533 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/6a3593c7d62b444d83af243bbce849c9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a3593c7d62b444d83af243bbce849c9 2024-11-07T12:36:23,541 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into 6a3593c7d62b444d83af243bbce849c9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:23,541 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:23,541 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=13, startTime=1730982983428; duration=0sec 2024-11-07T12:36:23,541 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:23,541 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:23,541 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:23,542 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/057db0716ae148a697ae24a243ca390b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/057db0716ae148a697ae24a243ca390b 2024-11-07T12:36:23,543 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:23,543 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:23,543 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:23,543 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/09b6e84f3b8546359abae7d6f0a444aa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/6511d5b9983549e697e96b83faff70fd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/21e165e0832e4283a79b62bdb912b779] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=36.9 K 2024-11-07T12:36:23,544 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09b6e84f3b8546359abae7d6f0a444aa, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730982979129 2024-11-07T12:36:23,545 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6511d5b9983549e697e96b83faff70fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730982980286 2024-11-07T12:36:23,546 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e165e0832e4283a79b62bdb912b779, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730982981444 2024-11-07T12:36:23,553 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 057db0716ae148a697ae24a243ca390b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:23,553 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:23,553 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=13, startTime=1730982983428; duration=0sec 2024-11-07T12:36:23,553 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:23,553 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:23,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741926_1102 (size=12301) 2024-11-07T12:36:23,562 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#90 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:23,562 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/e4a377894ed24db59af3a67108f81c85 is 50, key is test_row_0/C:col10/1730982981446/Put/seqid=0 2024-11-07T12:36:23,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741927_1103 (size=13289) 2024-11-07T12:36:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:23,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:23,611 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/e4a377894ed24db59af3a67108f81c85 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e4a377894ed24db59af3a67108f81c85 2024-11-07T12:36:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:36:23,620 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into e4a377894ed24db59af3a67108f81c85(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:23,620 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:23,620 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=13, startTime=1730982983428; duration=0sec 2024-11-07T12:36:23,620 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:23,620 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:23,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983043648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983043648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983043651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983043651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983043651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983043757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983043768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983043768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983043768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983043768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,961 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/62e7d2c8a5a949748be567e72c0dabbf 2024-11-07T12:36:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983043968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983043979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/94390a27b4a948d9a5ef89c88834dce7 is 50, key is test_row_0/B:col10/1730982982480/Put/seqid=0 2024-11-07T12:36:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983043979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983043979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741928_1104 (size=12301) 2024-11-07T12:36:23,987 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/94390a27b4a948d9a5ef89c88834dce7 2024-11-07T12:36:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983043987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f8ea9d3cfdcb4b50b8f720fb9759d784 is 50, key is test_row_0/C:col10/1730982982480/Put/seqid=0 2024-11-07T12:36:24,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741929_1105 (size=12301) 2024-11-07T12:36:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:36:24,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983044277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983044288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983044288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983044289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983044307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,408 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f8ea9d3cfdcb4b50b8f720fb9759d784 2024-11-07T12:36:24,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/62e7d2c8a5a949748be567e72c0dabbf as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/62e7d2c8a5a949748be567e72c0dabbf 2024-11-07T12:36:24,437 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/62e7d2c8a5a949748be567e72c0dabbf, entries=150, sequenceid=444, filesize=12.0 K 2024-11-07T12:36:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/94390a27b4a948d9a5ef89c88834dce7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/94390a27b4a948d9a5ef89c88834dce7 2024-11-07T12:36:24,451 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/94390a27b4a948d9a5ef89c88834dce7, entries=150, sequenceid=444, filesize=12.0 K 2024-11-07T12:36:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f8ea9d3cfdcb4b50b8f720fb9759d784 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f8ea9d3cfdcb4b50b8f720fb9759d784 2024-11-07T12:36:24,460 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f8ea9d3cfdcb4b50b8f720fb9759d784, entries=150, sequenceid=444, filesize=12.0 K 2024-11-07T12:36:24,464 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for f7b6889d440fde956272a8aab00142f0 in 987ms, sequenceid=444, compaction requested=false 2024-11-07T12:36:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-07T12:36:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-07T12:36:24,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T12:36:24,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-11-07T12:36:24,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4820 sec 2024-11-07T12:36:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:24,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T12:36:24,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983044805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983044805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983044809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983044811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983044818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/65975c1d3c0d494e99a59ed34ddfc92d is 50, key is test_row_0/A:col10/1730982984801/Put/seqid=0 2024-11-07T12:36:24,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741930_1106 (size=12301) 2024-11-07T12:36:24,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983044918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983044919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983044920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983044920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983045127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983045127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983045127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983045127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:36:25,138 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:25,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-07T12:36:25,141 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:36:25,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:25,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:36:25,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/65975c1d3c0d494e99a59ed34ddfc92d 2024-11-07T12:36:25,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/925713114c914f8db225c3e9084e7cc2 is 50, key is test_row_0/B:col10/1730982984801/Put/seqid=0 2024-11-07T12:36:25,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741931_1107 (size=12301) 2024-11-07T12:36:25,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/925713114c914f8db225c3e9084e7cc2 2024-11-07T12:36:25,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/e8c033ec5c4a4018b86d35d8b9ee834c is 50, key is test_row_0/C:col10/1730982984801/Put/seqid=0 2024-11-07T12:36:25,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:36:25,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:25,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741932_1108 (size=12301) 2024-11-07T12:36:25,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/e8c033ec5c4a4018b86d35d8b9ee834c 2024-11-07T12:36:25,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/65975c1d3c0d494e99a59ed34ddfc92d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/65975c1d3c0d494e99a59ed34ddfc92d 2024-11-07T12:36:25,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/65975c1d3c0d494e99a59ed34ddfc92d, entries=150, sequenceid=478, filesize=12.0 K 2024-11-07T12:36:25,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/925713114c914f8db225c3e9084e7cc2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/925713114c914f8db225c3e9084e7cc2 2024-11-07T12:36:25,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/925713114c914f8db225c3e9084e7cc2, entries=150, sequenceid=478, filesize=12.0 K 2024-11-07T12:36:25,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/e8c033ec5c4a4018b86d35d8b9ee834c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e8c033ec5c4a4018b86d35d8b9ee834c 2024-11-07T12:36:25,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e8c033ec5c4a4018b86d35d8b9ee834c, entries=150, sequenceid=478, filesize=12.0 K 2024-11-07T12:36:25,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for f7b6889d440fde956272a8aab00142f0 in 540ms, sequenceid=478, compaction requested=true 2024-11-07T12:36:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:25,342 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:36:25,343 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:25,343 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:25,343 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:25,343 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,344 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/057db0716ae148a697ae24a243ca390b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/62e7d2c8a5a949748be567e72c0dabbf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/65975c1d3c0d494e99a59ed34ddfc92d] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=37.0 K 2024-11-07T12:36:25,344 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057db0716ae148a697ae24a243ca390b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730982981444 2024-11-07T12:36:25,345 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e7d2c8a5a949748be567e72c0dabbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1730982982474 2024-11-07T12:36:25,345 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:25,345 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:25,345 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,345 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a3593c7d62b444d83af243bbce849c9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/94390a27b4a948d9a5ef89c88834dce7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/925713114c914f8db225c3e9084e7cc2] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=37.0 K 2024-11-07T12:36:25,345 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65975c1d3c0d494e99a59ed34ddfc92d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1730982984800 2024-11-07T12:36:25,346 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3593c7d62b444d83af243bbce849c9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730982981444 2024-11-07T12:36:25,346 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 94390a27b4a948d9a5ef89c88834dce7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1730982982474 2024-11-07T12:36:25,347 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 925713114c914f8db225c3e9084e7cc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1730982984800 2024-11-07T12:36:25,366 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:25,367 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/1592d01ed5ba4b0db5747847fb58d4ac is 50, key is test_row_0/A:col10/1730982984801/Put/seqid=0 2024-11-07T12:36:25,375 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:25,375 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/0bed502ccb4b49fcb7b6565d7a79777d is 50, key is test_row_0/B:col10/1730982984801/Put/seqid=0 2024-11-07T12:36:25,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741934_1110 (size=13391) 2024-11-07T12:36:25,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741933_1109 (size=13391) 2024-11-07T12:36:25,406 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/1592d01ed5ba4b0db5747847fb58d4ac as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1592d01ed5ba4b0db5747847fb58d4ac 2024-11-07T12:36:25,413 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 1592d01ed5ba4b0db5747847fb58d4ac(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:25,414 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:25,414 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=13, startTime=1730982985342; duration=0sec 2024-11-07T12:36:25,414 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:25,414 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:25,414 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:25,415 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:25,416 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:25,416 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,416 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e4a377894ed24db59af3a67108f81c85, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f8ea9d3cfdcb4b50b8f720fb9759d784, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e8c033ec5c4a4018b86d35d8b9ee834c] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=37.0 K 2024-11-07T12:36:25,416 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4a377894ed24db59af3a67108f81c85, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730982981444 2024-11-07T12:36:25,421 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8ea9d3cfdcb4b50b8f720fb9759d784, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1730982982474 2024-11-07T12:36:25,421 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c033ec5c4a4018b86d35d8b9ee834c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1730982984800 2024-11-07T12:36:25,431 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:25,432 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/9ff75a9b2863401f9ce79dd6ffab26e7 is 50, key is test_row_0/C:col10/1730982984801/Put/seqid=0 2024-11-07T12:36:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:25,444 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:25,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:36:25,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:36:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/bb917d83694e434d9aa0dc15ab13e080 is 50, key is test_row_0/A:col10/1730982985442/Put/seqid=0 2024-11-07T12:36:25,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741935_1111 (size=13391) 2024-11-07T12:36:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:36:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741936_1112 (size=19621) 2024-11-07T12:36:25,471 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/9ff75a9b2863401f9ce79dd6ffab26e7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9ff75a9b2863401f9ce79dd6ffab26e7 2024-11-07T12:36:25,484 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into 9ff75a9b2863401f9ce79dd6ffab26e7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:25,485 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:25,485 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=13, startTime=1730982985342; duration=0sec 2024-11-07T12:36:25,485 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:25,485 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:25,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983045480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983045482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983045484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983045487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983045598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983045598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983045599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983045599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:36:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:36:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:36:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,798 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/0bed502ccb4b49fcb7b6565d7a79777d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0bed502ccb4b49fcb7b6565d7a79777d 2024-11-07T12:36:25,805 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into 0bed502ccb4b49fcb7b6565d7a79777d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:25,805 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:25,805 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=13, startTime=1730982985342; duration=0sec 2024-11-07T12:36:25,805 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:25,805 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:25,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983045807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983045807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983045808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983045808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45144 deadline: 1730983045837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:25,839 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in f7b6889d440fde956272a8aab00142f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:36:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:25,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/bb917d83694e434d9aa0dc15ab13e080 2024-11-07T12:36:25,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/61cccd3576bf47a0a3c2de44710ccb02 is 50, key is test_row_0/B:col10/1730982985442/Put/seqid=0 2024-11-07T12:36:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741937_1113 (size=12301) 2024-11-07T12:36:25,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/61cccd3576bf47a0a3c2de44710ccb02 2024-11-07T12:36:25,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f984ef7b142b48e591eb9b7d5062c054 is 50, key is test_row_0/C:col10/1730982985442/Put/seqid=0 2024-11-07T12:36:25,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741938_1114 (size=12301) 2024-11-07T12:36:25,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f984ef7b142b48e591eb9b7d5062c054 2024-11-07T12:36:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/bb917d83694e434d9aa0dc15ab13e080 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/bb917d83694e434d9aa0dc15ab13e080 2024-11-07T12:36:25,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:36:25,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:25,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:25,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:25,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/bb917d83694e434d9aa0dc15ab13e080, entries=300, sequenceid=492, filesize=19.2 K 2024-11-07T12:36:25,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/61cccd3576bf47a0a3c2de44710ccb02 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/61cccd3576bf47a0a3c2de44710ccb02 2024-11-07T12:36:25,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/61cccd3576bf47a0a3c2de44710ccb02, entries=150, sequenceid=492, filesize=12.0 K 2024-11-07T12:36:25,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f984ef7b142b48e591eb9b7d5062c054 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f984ef7b142b48e591eb9b7d5062c054 2024-11-07T12:36:25,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f984ef7b142b48e591eb9b7d5062c054, entries=150, sequenceid=492, filesize=12.0 K 2024-11-07T12:36:25,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7b6889d440fde956272a8aab00142f0 in 492ms, sequenceid=492, compaction requested=false 2024-11-07T12:36:25,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:26,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:36:26,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:26,074 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:36:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/18bdb62deb234cd089c3b5efdfae1391 is 50, key is test_row_0/A:col10/1730982985477/Put/seqid=0 2024-11-07T12:36:26,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741939_1115 (size=12301) 2024-11-07T12:36:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:26,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. as already flushing 2024-11-07T12:36:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983046128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983046132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983046133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983046134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983046237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983046248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983046248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983046248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:36:26,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983046447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983046467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983046477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983046477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,496 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/18bdb62deb234cd089c3b5efdfae1391 2024-11-07T12:36:26,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/cae771d30d534a2e9501c1861ad67f12 is 50, key is test_row_0/B:col10/1730982985477/Put/seqid=0 2024-11-07T12:36:26,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741940_1116 (size=12301) 2024-11-07T12:36:26,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983046757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983046778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983046787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983046787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:26,910 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/cae771d30d534a2e9501c1861ad67f12 2024-11-07T12:36:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f51663cd20b64ac68758305142055e9f is 50, key is test_row_0/C:col10/1730982985477/Put/seqid=0 2024-11-07T12:36:26,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741941_1117 (size=12301) 2024-11-07T12:36:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983047267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:36:27,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983047287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:27,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983047298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:27,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983047298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:27,346 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f51663cd20b64ac68758305142055e9f 2024-11-07T12:36:27,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/18bdb62deb234cd089c3b5efdfae1391 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/18bdb62deb234cd089c3b5efdfae1391 2024-11-07T12:36:27,367 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/18bdb62deb234cd089c3b5efdfae1391, entries=150, sequenceid=517, filesize=12.0 K 2024-11-07T12:36:27,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/cae771d30d534a2e9501c1861ad67f12 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/cae771d30d534a2e9501c1861ad67f12 2024-11-07T12:36:27,378 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/cae771d30d534a2e9501c1861ad67f12, entries=150, sequenceid=517, filesize=12.0 K 2024-11-07T12:36:27,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f51663cd20b64ac68758305142055e9f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f51663cd20b64ac68758305142055e9f 2024-11-07T12:36:27,392 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f51663cd20b64ac68758305142055e9f, entries=150, sequenceid=517, filesize=12.0 K 2024-11-07T12:36:27,394 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for f7b6889d440fde956272a8aab00142f0 in 1320ms, sequenceid=517, compaction requested=true 2024-11-07T12:36:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-07T12:36:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-07T12:36:27,402 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T12:36:27,402 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2530 sec 2024-11-07T12:36:27,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.2660 sec 2024-11-07T12:36:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:27,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:27,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:36:27,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/e2114b283ebc40efbb449eba5635e154 is 50, key is test_row_0/A:col10/1730982986132/Put/seqid=0 2024-11-07T12:36:27,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741942_1118 (size=12301) 2024-11-07T12:36:27,994 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-07T12:36:27,994 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:27,995 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-07T12:36:27,995 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-07T12:36:27,995 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:27,995 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:27,995 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-07T12:36:27,995 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:27,996 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-07T12:36:27,996 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:28,001 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,001 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,001 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,001 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,002 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,002 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:28,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:28,003 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:28,003 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,003 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:28,003 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:28,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:28,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45158 deadline: 1730983048278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:28,279 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in f7b6889d440fde956272a8aab00142f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:36:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:28,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45146 deadline: 1730983048297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:28,298 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in f7b6889d440fde956272a8aab00142f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:36:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:28,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1730983048307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:28,309 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in f7b6889d440fde956272a8aab00142f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:36:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1730983048318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:28,319 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in f7b6889d440fde956272a8aab00142f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:36:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:36:28,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/e2114b283ebc40efbb449eba5635e154 2024-11-07T12:36:28,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3b9e515688ee46d6805ee2a07ca1f9eb is 50, key is test_row_0/B:col10/1730982986132/Put/seqid=0 2024-11-07T12:36:28,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741943_1119 (size=12301) 2024-11-07T12:36:28,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3b9e515688ee46d6805ee2a07ca1f9eb 2024-11-07T12:36:28,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/3e19ec07001d496f9077e5055d7ff185 is 50, key is test_row_0/C:col10/1730982986132/Put/seqid=0 2024-11-07T12:36:28,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741944_1120 (size=12301) 2024-11-07T12:36:29,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/3e19ec07001d496f9077e5055d7ff185 2024-11-07T12:36:29,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/e2114b283ebc40efbb449eba5635e154 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e2114b283ebc40efbb449eba5635e154 2024-11-07T12:36:29,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e2114b283ebc40efbb449eba5635e154, entries=150, sequenceid=532, filesize=12.0 K 2024-11-07T12:36:29,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/3b9e515688ee46d6805ee2a07ca1f9eb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3b9e515688ee46d6805ee2a07ca1f9eb 2024-11-07T12:36:29,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3b9e515688ee46d6805ee2a07ca1f9eb, entries=150, sequenceid=532, filesize=12.0 K 2024-11-07T12:36:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/3e19ec07001d496f9077e5055d7ff185 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/3e19ec07001d496f9077e5055d7ff185 2024-11-07T12:36:29,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/3e19ec07001d496f9077e5055d7ff185, entries=150, sequenceid=532, filesize=12.0 K 2024-11-07T12:36:29,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f7b6889d440fde956272a8aab00142f0 in 1317ms, sequenceid=532, compaction requested=true 2024-11-07T12:36:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:29,182 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:29,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:29,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:29,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7b6889d440fde956272a8aab00142f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:29,183 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:29,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:29,184 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:29,184 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:29,184 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/B is initiating minor compaction (all files) 2024-11-07T12:36:29,184 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/A is initiating minor compaction (all files) 2024-11-07T12:36:29,184 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/B in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:29,184 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/A in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:29,184 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1592d01ed5ba4b0db5747847fb58d4ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/bb917d83694e434d9aa0dc15ab13e080, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/18bdb62deb234cd089c3b5efdfae1391, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e2114b283ebc40efbb449eba5635e154] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=56.3 K 2024-11-07T12:36:29,184 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0bed502ccb4b49fcb7b6565d7a79777d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/61cccd3576bf47a0a3c2de44710ccb02, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/cae771d30d534a2e9501c1861ad67f12, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3b9e515688ee46d6805ee2a07ca1f9eb] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=49.1 K 2024-11-07T12:36:29,185 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bed502ccb4b49fcb7b6565d7a79777d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1730982984800 2024-11-07T12:36:29,185 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1592d01ed5ba4b0db5747847fb58d4ac, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1730982984800 2024-11-07T12:36:29,186 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 61cccd3576bf47a0a3c2de44710ccb02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1730982985442 2024-11-07T12:36:29,186 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb917d83694e434d9aa0dc15ab13e080, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1730982985439 2024-11-07T12:36:29,186 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting cae771d30d534a2e9501c1861ad67f12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730982985473 2024-11-07T12:36:29,186 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18bdb62deb234cd089c3b5efdfae1391, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730982985473 2024-11-07T12:36:29,187 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b9e515688ee46d6805ee2a07ca1f9eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1730982986126 2024-11-07T12:36:29,187 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2114b283ebc40efbb449eba5635e154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1730982986126 2024-11-07T12:36:29,210 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#A#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:29,211 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/55fdf7fb2a964e1b8b08891c61ac34b0 is 50, key is test_row_0/A:col10/1730982986132/Put/seqid=0 2024-11-07T12:36:29,214 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#B#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:29,215 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/87e3059635c541a286dd7a7b7ea8b6f4 is 50, key is test_row_0/B:col10/1730982986132/Put/seqid=0 2024-11-07T12:36:29,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741946_1122 (size=13527) 2024-11-07T12:36:29,238 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/87e3059635c541a286dd7a7b7ea8b6f4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/87e3059635c541a286dd7a7b7ea8b6f4 2024-11-07T12:36:29,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741945_1121 (size=13527) 2024-11-07T12:36:29,248 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/B of f7b6889d440fde956272a8aab00142f0 into 87e3059635c541a286dd7a7b7ea8b6f4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:29,248 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:29,248 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/B, priority=12, startTime=1730982989183; duration=0sec 2024-11-07T12:36:29,248 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:29,249 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:B 2024-11-07T12:36:29,249 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:29,250 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:29,250 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): f7b6889d440fde956272a8aab00142f0/C is initiating minor compaction (all files) 2024-11-07T12:36:29,250 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7b6889d440fde956272a8aab00142f0/C in TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:29,250 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9ff75a9b2863401f9ce79dd6ffab26e7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f984ef7b142b48e591eb9b7d5062c054, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f51663cd20b64ac68758305142055e9f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/3e19ec07001d496f9077e5055d7ff185] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp, totalSize=49.1 K 2024-11-07T12:36:29,250 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ff75a9b2863401f9ce79dd6ffab26e7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1730982984800 2024-11-07T12:36:29,251 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f984ef7b142b48e591eb9b7d5062c054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1730982985442 2024-11-07T12:36:29,251 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f51663cd20b64ac68758305142055e9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730982985473 2024-11-07T12:36:29,251 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e19ec07001d496f9077e5055d7ff185, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1730982986126 2024-11-07T12:36:29,262 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7b6889d440fde956272a8aab00142f0#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:29,262 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f74f6f4c1dd04ab5b979116f789d53c9 is 50, key is test_row_0/C:col10/1730982986132/Put/seqid=0 2024-11-07T12:36:29,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741947_1123 (size=13527) 2024-11-07T12:36:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:36:29,308 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:29,653 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/55fdf7fb2a964e1b8b08891c61ac34b0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55fdf7fb2a964e1b8b08891c61ac34b0 2024-11-07T12:36:29,658 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/A of f7b6889d440fde956272a8aab00142f0 into 55fdf7fb2a964e1b8b08891c61ac34b0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:29,658 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:29,658 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/A, priority=12, startTime=1730982989182; duration=0sec 2024-11-07T12:36:29,659 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:29,659 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:A 2024-11-07T12:36:29,676 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/f74f6f4c1dd04ab5b979116f789d53c9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f74f6f4c1dd04ab5b979116f789d53c9 2024-11-07T12:36:29,682 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f7b6889d440fde956272a8aab00142f0/C of f7b6889d440fde956272a8aab00142f0 into f74f6f4c1dd04ab5b979116f789d53c9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:29,682 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:29,682 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., storeName=f7b6889d440fde956272a8aab00142f0/C, priority=12, startTime=1730982989183; duration=0sec 2024-11-07T12:36:29,682 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:29,682 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7b6889d440fde956272a8aab00142f0:C 2024-11-07T12:36:30,246 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:36:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:30,309 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:30,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:36:30,309 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-07T12:36:30,309 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:30,309 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,309 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-07T12:36:30,310 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,310 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:30,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:30,310 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,310 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:30,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5def71e6b43e42e39692b0d6988cb92c is 50, key is test_row_0/A:col10/1730982990308/Put/seqid=0 2024-11-07T12:36:30,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741948_1124 (size=12301) 2024-11-07T12:36:30,321 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-07T12:36:30,321 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:30,321 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,322 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:30,349 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-07T12:36:30,349 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:30,349 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,349 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:30,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2828 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2758 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1273 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3816 rows 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1268 2024-11-07T12:36:30,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3804 rows 2024-11-07T12:36:30,350 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:36:30,350 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:36:30,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:30,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:30,354 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T12:36:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:36:30,363 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982990362"}]},"ts":"1730982990362"} 2024-11-07T12:36:30,365 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:36:30,365 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:36:30,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:36:30,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, UNASSIGN}] 2024-11-07T12:36:30,372 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, UNASSIGN 2024-11-07T12:36:30,373 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f7b6889d440fde956272a8aab00142f0, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:30,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, UNASSIGN because future has completed 2024-11-07T12:36:30,376 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:36:30,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:36:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:36:30,533 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:30,534 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:36:30,534 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing f7b6889d440fde956272a8aab00142f0, disabling compactions & flushes 2024-11-07T12:36:30,535 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:36:30,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5def71e6b43e42e39692b0d6988cb92c 2024-11-07T12:36:30,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/0f152c21a7264e1e9cb1769c491ecbe4 is 50, key is test_row_0/B:col10/1730982990308/Put/seqid=0 2024-11-07T12:36:30,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741949_1125 (size=12301) 2024-11-07T12:36:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:36:31,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/0f152c21a7264e1e9cb1769c491ecbe4 2024-11-07T12:36:31,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/85068c2245744c5ea2b6eb51f87a7de1 is 50, key is test_row_0/C:col10/1730982990308/Put/seqid=0 2024-11-07T12:36:31,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741950_1126 (size=12301) 2024-11-07T12:36:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:36:31,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/85068c2245744c5ea2b6eb51f87a7de1 2024-11-07T12:36:31,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/5def71e6b43e42e39692b0d6988cb92c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5def71e6b43e42e39692b0d6988cb92c 2024-11-07T12:36:31,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5def71e6b43e42e39692b0d6988cb92c, entries=150, sequenceid=558, filesize=12.0 K 2024-11-07T12:36:31,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/0f152c21a7264e1e9cb1769c491ecbe4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0f152c21a7264e1e9cb1769c491ecbe4 2024-11-07T12:36:31,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0f152c21a7264e1e9cb1769c491ecbe4, entries=150, sequenceid=558, filesize=12.0 K 2024-11-07T12:36:31,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/85068c2245744c5ea2b6eb51f87a7de1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/85068c2245744c5ea2b6eb51f87a7de1 2024-11-07T12:36:31,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/85068c2245744c5ea2b6eb51f87a7de1, entries=150, sequenceid=558, filesize=12.0 K 2024-11-07T12:36:31,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=13.42 KB/13740 for f7b6889d440fde956272a8aab00142f0 in 1258ms, sequenceid=558, compaction requested=false 2024-11-07T12:36:31,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7b6889d440fde956272a8aab00142f0: 2024-11-07T12:36:31,567 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:31,567 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:31,568 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. after waiting 0 ms 2024-11-07T12:36:31,568 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:31,568 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing f7b6889d440fde956272a8aab00142f0 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-07T12:36:31,571 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/233d9caa0a9b49ecba2e217ee08e85a6 is 50, key is test_row_0/A:col10/1730982990348/Put/seqid=0 2024-11-07T12:36:31,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741951_1127 (size=9857) 2024-11-07T12:36:31,976 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/233d9caa0a9b49ecba2e217ee08e85a6 2024-11-07T12:36:31,984 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/6a1d6db4460245b687b5f725a7375d9a is 50, key is test_row_0/B:col10/1730982990348/Put/seqid=0 2024-11-07T12:36:31,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741952_1128 (size=9857) 2024-11-07T12:36:32,389 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/6a1d6db4460245b687b5f725a7375d9a 2024-11-07T12:36:32,397 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/84da1b6fa5a14d1f80bb511c384ddde8 is 50, key is test_row_0/C:col10/1730982990348/Put/seqid=0 2024-11-07T12:36:32,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741953_1129 (size=9857) 2024-11-07T12:36:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:36:32,802 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/84da1b6fa5a14d1f80bb511c384ddde8 2024-11-07T12:36:32,807 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/A/233d9caa0a9b49ecba2e217ee08e85a6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/233d9caa0a9b49ecba2e217ee08e85a6 2024-11-07T12:36:32,811 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/233d9caa0a9b49ecba2e217ee08e85a6, entries=100, sequenceid=563, filesize=9.6 K 2024-11-07T12:36:32,812 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/B/6a1d6db4460245b687b5f725a7375d9a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a1d6db4460245b687b5f725a7375d9a 2024-11-07T12:36:32,816 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a1d6db4460245b687b5f725a7375d9a, entries=100, sequenceid=563, filesize=9.6 K 2024-11-07T12:36:32,816 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/.tmp/C/84da1b6fa5a14d1f80bb511c384ddde8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/84da1b6fa5a14d1f80bb511c384ddde8 2024-11-07T12:36:32,820 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/84da1b6fa5a14d1f80bb511c384ddde8, entries=100, sequenceid=563, filesize=9.6 K 2024-11-07T12:36:32,821 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for f7b6889d440fde956272a8aab00142f0 in 1253ms, sequenceid=563, compaction requested=true 2024-11-07T12:36:32,822 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3c4ef4e42cae41538f9ee1048159479f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/0ff3c41962cb4ad387a6cd00e427dfdb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55cae0d8207546609bc3899ba0ad5cb5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3298618afc5849bb8ffbeac075f76961, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/87d6e4ab15c94c86837001751e20b112, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/cab9b9f6737e4cd0acf1bdd2a0610797, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/382af1242a7c4fd2a6efe7cfae7ad475, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/294d224ab402482f9534b4c2c722046a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5f6a00582619480889071aa40f09cd18, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1c04a1807664422ca9292fa3080367f2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c04e77e3f55c4be3b01f3e651dcfa7c9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/a227cf0ef0aa4c4da9069834353862ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7f30af4d043241239ebb429cf73980c4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/91645f745abd4efbbf3036e4ef878e97, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/d9736de1601a4e39bf864ecd3501f81d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3f60a04baccf4b489cb1cf6a9cab291d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/9720cf4529304a86abcef3b29e430613, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c5e5f816d9164976ac7cd3028c6176c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/830b8f022ac7447eae018e64df8a80b2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3880a425c53a404b877286f5153209d4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/405798dc677e464089978a0dea3ea830, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8a5cbbd9c6474e3d9fa902824ba260e4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7170b4aed886410aaf19596469751a93, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8ae030583ae641308202da5f33af753a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5717b01ba0184038b83b12a80e7dadf1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/001e8ecacea64dcab39b67afc705701f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/90a79d42755a413a8de0194d209fc8fa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e9fdbef529f64b3ba1520c9313958011, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/16202eb01c204dd291cd1d9386c5f005, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/057db0716ae148a697ae24a243ca390b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/62e7d2c8a5a949748be567e72c0dabbf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1592d01ed5ba4b0db5747847fb58d4ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/65975c1d3c0d494e99a59ed34ddfc92d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/bb917d83694e434d9aa0dc15ab13e080, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/18bdb62deb234cd089c3b5efdfae1391, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e2114b283ebc40efbb449eba5635e154] to archive 2024-11-07T12:36:32,825 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:36:32,831 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3c4ef4e42cae41538f9ee1048159479f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3c4ef4e42cae41538f9ee1048159479f 2024-11-07T12:36:32,832 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/0ff3c41962cb4ad387a6cd00e427dfdb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/0ff3c41962cb4ad387a6cd00e427dfdb 2024-11-07T12:36:32,834 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55cae0d8207546609bc3899ba0ad5cb5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55cae0d8207546609bc3899ba0ad5cb5 2024-11-07T12:36:32,835 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3298618afc5849bb8ffbeac075f76961 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3298618afc5849bb8ffbeac075f76961 2024-11-07T12:36:32,836 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/87d6e4ab15c94c86837001751e20b112 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/87d6e4ab15c94c86837001751e20b112 2024-11-07T12:36:32,838 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/cab9b9f6737e4cd0acf1bdd2a0610797 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/cab9b9f6737e4cd0acf1bdd2a0610797 2024-11-07T12:36:32,839 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/382af1242a7c4fd2a6efe7cfae7ad475 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/382af1242a7c4fd2a6efe7cfae7ad475 2024-11-07T12:36:32,841 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/294d224ab402482f9534b4c2c722046a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/294d224ab402482f9534b4c2c722046a 2024-11-07T12:36:32,842 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5f6a00582619480889071aa40f09cd18 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5f6a00582619480889071aa40f09cd18 2024-11-07T12:36:32,847 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1c04a1807664422ca9292fa3080367f2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1c04a1807664422ca9292fa3080367f2 2024-11-07T12:36:32,848 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c04e77e3f55c4be3b01f3e651dcfa7c9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c04e77e3f55c4be3b01f3e651dcfa7c9 2024-11-07T12:36:32,850 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/a227cf0ef0aa4c4da9069834353862ac to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/a227cf0ef0aa4c4da9069834353862ac 2024-11-07T12:36:32,851 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7f30af4d043241239ebb429cf73980c4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7f30af4d043241239ebb429cf73980c4 2024-11-07T12:36:32,853 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/91645f745abd4efbbf3036e4ef878e97 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/91645f745abd4efbbf3036e4ef878e97 2024-11-07T12:36:32,854 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/d9736de1601a4e39bf864ecd3501f81d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/d9736de1601a4e39bf864ecd3501f81d 2024-11-07T12:36:32,856 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3f60a04baccf4b489cb1cf6a9cab291d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3f60a04baccf4b489cb1cf6a9cab291d 2024-11-07T12:36:32,857 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/9720cf4529304a86abcef3b29e430613 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/9720cf4529304a86abcef3b29e430613 2024-11-07T12:36:32,858 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c5e5f816d9164976ac7cd3028c6176c3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/c5e5f816d9164976ac7cd3028c6176c3 2024-11-07T12:36:32,859 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/830b8f022ac7447eae018e64df8a80b2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/830b8f022ac7447eae018e64df8a80b2 2024-11-07T12:36:32,861 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3880a425c53a404b877286f5153209d4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/3880a425c53a404b877286f5153209d4 2024-11-07T12:36:32,862 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/405798dc677e464089978a0dea3ea830 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/405798dc677e464089978a0dea3ea830 2024-11-07T12:36:32,863 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8a5cbbd9c6474e3d9fa902824ba260e4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8a5cbbd9c6474e3d9fa902824ba260e4 2024-11-07T12:36:32,864 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7170b4aed886410aaf19596469751a93 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/7170b4aed886410aaf19596469751a93 2024-11-07T12:36:32,866 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8ae030583ae641308202da5f33af753a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/8ae030583ae641308202da5f33af753a 2024-11-07T12:36:32,867 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5717b01ba0184038b83b12a80e7dadf1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5717b01ba0184038b83b12a80e7dadf1 2024-11-07T12:36:32,868 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/001e8ecacea64dcab39b67afc705701f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/001e8ecacea64dcab39b67afc705701f 2024-11-07T12:36:32,869 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/90a79d42755a413a8de0194d209fc8fa to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/90a79d42755a413a8de0194d209fc8fa 2024-11-07T12:36:32,871 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e9fdbef529f64b3ba1520c9313958011 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e9fdbef529f64b3ba1520c9313958011 2024-11-07T12:36:32,872 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/16202eb01c204dd291cd1d9386c5f005 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/16202eb01c204dd291cd1d9386c5f005 2024-11-07T12:36:32,873 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/057db0716ae148a697ae24a243ca390b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/057db0716ae148a697ae24a243ca390b 2024-11-07T12:36:32,874 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/62e7d2c8a5a949748be567e72c0dabbf to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/62e7d2c8a5a949748be567e72c0dabbf 2024-11-07T12:36:32,875 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1592d01ed5ba4b0db5747847fb58d4ac to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/1592d01ed5ba4b0db5747847fb58d4ac 2024-11-07T12:36:32,876 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/65975c1d3c0d494e99a59ed34ddfc92d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/65975c1d3c0d494e99a59ed34ddfc92d 2024-11-07T12:36:32,877 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/bb917d83694e434d9aa0dc15ab13e080 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/bb917d83694e434d9aa0dc15ab13e080 2024-11-07T12:36:32,878 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/18bdb62deb234cd089c3b5efdfae1391 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/18bdb62deb234cd089c3b5efdfae1391 2024-11-07T12:36:32,879 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e2114b283ebc40efbb449eba5635e154 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/e2114b283ebc40efbb449eba5635e154 2024-11-07T12:36:32,897 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b55e24593c4d47bf86ff8dd3df51b63b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b9b8a15a556442ccbf4e4773497e334b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3276cda3d08f46609c83cd7afaf47cb1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/a788ca6681ce44fbb622a7cca8e106ab, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/519d18c65f684e818dfeca2ee0d2535c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d7714f44a37049adb84cd14dc7702f74, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7e2eca51c57845a5bdaaccd9772a734d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7f2b28a39d794e98a7c24fe65e0d8aad, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4cfdfc00db1c4676b0f213cf7fd7d2de, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2ad7e5dba7ab42ff8a2bb64af5d72c14, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d32051386d154f3c9f3367e06a7a8ffa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4692c935e13143fe8e010de34f4ef595, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/c1db7cae42124d8faeb016d85cfc9d12, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/e7818644c563491dbb7adddbfb61171b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d611acd99aaa4639b661dbb63756f9fa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/935aa85b9ee74f4fb91c37f35ad49a61, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9ba9bc8eb93247a6bd05dda91003dc1a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2153b82e98334a7bac222d0b24650b50, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/f530645b372c4070bbad7d12a3674e6e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/56a5c2d95ed146a5b22c96a4ba68fcc9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3337ce5eb3e145d69c90bc83fca3349b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/5e6284a86db54d649e31b3377d741d43, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9fc78477344b4dc19cae9b7cd4717dd7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9f2f6dd7202943a8b416268cf452d667, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4b60aff464b54753b59e38648b2b96a0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3d7db9bf993f4584be74fbefa6e20733, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/bc070fe1f2a1417e90ad59dbfc1e68c0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2083d2fe8acf43bd85f2c5afb62223a7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a3593c7d62b444d83af243bbce849c9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/226ca194a2b34b3181695018ce864168, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/94390a27b4a948d9a5ef89c88834dce7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0bed502ccb4b49fcb7b6565d7a79777d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/925713114c914f8db225c3e9084e7cc2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/61cccd3576bf47a0a3c2de44710ccb02, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/cae771d30d534a2e9501c1861ad67f12, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3b9e515688ee46d6805ee2a07ca1f9eb] to archive 2024-11-07T12:36:32,898 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:36:32,900 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b55e24593c4d47bf86ff8dd3df51b63b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b55e24593c4d47bf86ff8dd3df51b63b 2024-11-07T12:36:32,902 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b9b8a15a556442ccbf4e4773497e334b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/b9b8a15a556442ccbf4e4773497e334b 2024-11-07T12:36:32,903 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3276cda3d08f46609c83cd7afaf47cb1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3276cda3d08f46609c83cd7afaf47cb1 2024-11-07T12:36:32,904 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/a788ca6681ce44fbb622a7cca8e106ab to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/a788ca6681ce44fbb622a7cca8e106ab 2024-11-07T12:36:32,906 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/519d18c65f684e818dfeca2ee0d2535c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/519d18c65f684e818dfeca2ee0d2535c 2024-11-07T12:36:32,907 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d7714f44a37049adb84cd14dc7702f74 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d7714f44a37049adb84cd14dc7702f74 2024-11-07T12:36:32,908 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7e2eca51c57845a5bdaaccd9772a734d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7e2eca51c57845a5bdaaccd9772a734d 2024-11-07T12:36:32,910 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7f2b28a39d794e98a7c24fe65e0d8aad to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/7f2b28a39d794e98a7c24fe65e0d8aad 2024-11-07T12:36:32,911 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4cfdfc00db1c4676b0f213cf7fd7d2de to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4cfdfc00db1c4676b0f213cf7fd7d2de 2024-11-07T12:36:32,912 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2ad7e5dba7ab42ff8a2bb64af5d72c14 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2ad7e5dba7ab42ff8a2bb64af5d72c14 2024-11-07T12:36:32,913 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d32051386d154f3c9f3367e06a7a8ffa to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d32051386d154f3c9f3367e06a7a8ffa 2024-11-07T12:36:32,914 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4692c935e13143fe8e010de34f4ef595 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4692c935e13143fe8e010de34f4ef595 2024-11-07T12:36:32,915 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/c1db7cae42124d8faeb016d85cfc9d12 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/c1db7cae42124d8faeb016d85cfc9d12 2024-11-07T12:36:32,916 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/e7818644c563491dbb7adddbfb61171b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/e7818644c563491dbb7adddbfb61171b 2024-11-07T12:36:32,918 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d611acd99aaa4639b661dbb63756f9fa to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/d611acd99aaa4639b661dbb63756f9fa 2024-11-07T12:36:32,919 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/935aa85b9ee74f4fb91c37f35ad49a61 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/935aa85b9ee74f4fb91c37f35ad49a61 2024-11-07T12:36:32,920 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9ba9bc8eb93247a6bd05dda91003dc1a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9ba9bc8eb93247a6bd05dda91003dc1a 2024-11-07T12:36:32,921 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2153b82e98334a7bac222d0b24650b50 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2153b82e98334a7bac222d0b24650b50 2024-11-07T12:36:32,923 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/f530645b372c4070bbad7d12a3674e6e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/f530645b372c4070bbad7d12a3674e6e 2024-11-07T12:36:32,924 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/56a5c2d95ed146a5b22c96a4ba68fcc9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/56a5c2d95ed146a5b22c96a4ba68fcc9 2024-11-07T12:36:32,925 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3337ce5eb3e145d69c90bc83fca3349b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3337ce5eb3e145d69c90bc83fca3349b 2024-11-07T12:36:32,926 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/5e6284a86db54d649e31b3377d741d43 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/5e6284a86db54d649e31b3377d741d43 2024-11-07T12:36:32,928 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9fc78477344b4dc19cae9b7cd4717dd7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9fc78477344b4dc19cae9b7cd4717dd7 2024-11-07T12:36:32,929 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9f2f6dd7202943a8b416268cf452d667 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/9f2f6dd7202943a8b416268cf452d667 2024-11-07T12:36:32,930 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4b60aff464b54753b59e38648b2b96a0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/4b60aff464b54753b59e38648b2b96a0 2024-11-07T12:36:32,931 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3d7db9bf993f4584be74fbefa6e20733 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3d7db9bf993f4584be74fbefa6e20733 2024-11-07T12:36:32,932 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/bc070fe1f2a1417e90ad59dbfc1e68c0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/bc070fe1f2a1417e90ad59dbfc1e68c0 2024-11-07T12:36:32,933 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2083d2fe8acf43bd85f2c5afb62223a7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/2083d2fe8acf43bd85f2c5afb62223a7 2024-11-07T12:36:32,934 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a3593c7d62b444d83af243bbce849c9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a3593c7d62b444d83af243bbce849c9 2024-11-07T12:36:32,935 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/226ca194a2b34b3181695018ce864168 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/226ca194a2b34b3181695018ce864168 2024-11-07T12:36:32,936 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/94390a27b4a948d9a5ef89c88834dce7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/94390a27b4a948d9a5ef89c88834dce7 2024-11-07T12:36:32,937 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0bed502ccb4b49fcb7b6565d7a79777d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0bed502ccb4b49fcb7b6565d7a79777d 2024-11-07T12:36:32,938 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/925713114c914f8db225c3e9084e7cc2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/925713114c914f8db225c3e9084e7cc2 2024-11-07T12:36:32,939 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/61cccd3576bf47a0a3c2de44710ccb02 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/61cccd3576bf47a0a3c2de44710ccb02 2024-11-07T12:36:32,941 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/cae771d30d534a2e9501c1861ad67f12 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/cae771d30d534a2e9501c1861ad67f12 2024-11-07T12:36:32,942 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3b9e515688ee46d6805ee2a07ca1f9eb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/3b9e515688ee46d6805ee2a07ca1f9eb 2024-11-07T12:36:32,944 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/0912bd9049db4378aa488e85ce6e79ba, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c1201e51bfb644e2bb8faad993eddcca, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/8946d2ee109349a0986ba2f15b6d3c11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/36954efc2cd54b22a764f6be210c7221, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/fcf5c163b42541a5ab2dc11526f7b423, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/817abd67df864d06af3f0a2f77b3df37, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/cccd20d8476243939d98b472e2146e1f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b901fdf6cbd5423b8234ccc674b50702, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/358ef9e9e59e49ae89a92ecb4e80bfcb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1914fcfd9118477a9d5566873da73763, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/da9a6bd65ffb49b3b957fd2623de8dcd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c83ea9e1f0b5466f87e2d5d50f489bb9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1f8301357180432ba28bfb48435cbdc5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/62fea264893d492581a21cdc1398ec39, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b44d77e555c149ef8927af71943bb53b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e700a0c1865a43ce95a8dc57ce3ce218, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/821d161c93b74f1db465809089f46ef2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/137e624bc93d4292bfed656c7649caba, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9dcbec8e9f514430aa9b2d639a279dd8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/16c86a8160e742d6af61de976a8bb552, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/99b1ea81fdf0484eaef199daabb2d635, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/a5d730195be543c18d729ba98e9f363f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b98783020b304f818071df94165644c7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c4202498aefd42fe8d75f07918e44f11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/09b6e84f3b8546359abae7d6f0a444aa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c27ceb12a6e24013a96ccd5f84ebb62f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/6511d5b9983549e697e96b83faff70fd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e4a377894ed24db59af3a67108f81c85, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/21e165e0832e4283a79b62bdb912b779, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f8ea9d3cfdcb4b50b8f720fb9759d784, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9ff75a9b2863401f9ce79dd6ffab26e7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e8c033ec5c4a4018b86d35d8b9ee834c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f984ef7b142b48e591eb9b7d5062c054, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f51663cd20b64ac68758305142055e9f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/3e19ec07001d496f9077e5055d7ff185] to archive 2024-11-07T12:36:32,945 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:36:32,946 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/0912bd9049db4378aa488e85ce6e79ba to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/0912bd9049db4378aa488e85ce6e79ba 2024-11-07T12:36:32,948 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c1201e51bfb644e2bb8faad993eddcca to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c1201e51bfb644e2bb8faad993eddcca 2024-11-07T12:36:32,949 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/8946d2ee109349a0986ba2f15b6d3c11 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/8946d2ee109349a0986ba2f15b6d3c11 2024-11-07T12:36:32,951 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/36954efc2cd54b22a764f6be210c7221 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/36954efc2cd54b22a764f6be210c7221 2024-11-07T12:36:32,952 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/fcf5c163b42541a5ab2dc11526f7b423 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/fcf5c163b42541a5ab2dc11526f7b423 2024-11-07T12:36:32,953 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/817abd67df864d06af3f0a2f77b3df37 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/817abd67df864d06af3f0a2f77b3df37 2024-11-07T12:36:32,954 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/cccd20d8476243939d98b472e2146e1f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/cccd20d8476243939d98b472e2146e1f 2024-11-07T12:36:32,956 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b901fdf6cbd5423b8234ccc674b50702 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b901fdf6cbd5423b8234ccc674b50702 2024-11-07T12:36:32,957 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/358ef9e9e59e49ae89a92ecb4e80bfcb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/358ef9e9e59e49ae89a92ecb4e80bfcb 2024-11-07T12:36:32,958 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1914fcfd9118477a9d5566873da73763 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1914fcfd9118477a9d5566873da73763 2024-11-07T12:36:32,959 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/da9a6bd65ffb49b3b957fd2623de8dcd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/da9a6bd65ffb49b3b957fd2623de8dcd 2024-11-07T12:36:32,961 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c83ea9e1f0b5466f87e2d5d50f489bb9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c83ea9e1f0b5466f87e2d5d50f489bb9 2024-11-07T12:36:32,962 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1f8301357180432ba28bfb48435cbdc5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/1f8301357180432ba28bfb48435cbdc5 2024-11-07T12:36:32,963 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b2cfa5cd9d8242c0bdf88dbcd1885f9c 2024-11-07T12:36:32,965 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/62fea264893d492581a21cdc1398ec39 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/62fea264893d492581a21cdc1398ec39 2024-11-07T12:36:32,966 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b44d77e555c149ef8927af71943bb53b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b44d77e555c149ef8927af71943bb53b 2024-11-07T12:36:32,967 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e700a0c1865a43ce95a8dc57ce3ce218 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e700a0c1865a43ce95a8dc57ce3ce218 2024-11-07T12:36:32,969 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/821d161c93b74f1db465809089f46ef2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/821d161c93b74f1db465809089f46ef2 2024-11-07T12:36:32,970 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/137e624bc93d4292bfed656c7649caba to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/137e624bc93d4292bfed656c7649caba 2024-11-07T12:36:32,971 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9dcbec8e9f514430aa9b2d639a279dd8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9dcbec8e9f514430aa9b2d639a279dd8 2024-11-07T12:36:32,972 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/16c86a8160e742d6af61de976a8bb552 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/16c86a8160e742d6af61de976a8bb552 2024-11-07T12:36:32,974 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/99b1ea81fdf0484eaef199daabb2d635 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/99b1ea81fdf0484eaef199daabb2d635 2024-11-07T12:36:32,975 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/a5d730195be543c18d729ba98e9f363f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/a5d730195be543c18d729ba98e9f363f 2024-11-07T12:36:32,976 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b98783020b304f818071df94165644c7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/b98783020b304f818071df94165644c7 2024-11-07T12:36:32,977 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c4202498aefd42fe8d75f07918e44f11 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c4202498aefd42fe8d75f07918e44f11 2024-11-07T12:36:32,978 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/09b6e84f3b8546359abae7d6f0a444aa to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/09b6e84f3b8546359abae7d6f0a444aa 2024-11-07T12:36:32,979 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c27ceb12a6e24013a96ccd5f84ebb62f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/c27ceb12a6e24013a96ccd5f84ebb62f 2024-11-07T12:36:32,980 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/6511d5b9983549e697e96b83faff70fd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/6511d5b9983549e697e96b83faff70fd 2024-11-07T12:36:32,982 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e4a377894ed24db59af3a67108f81c85 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e4a377894ed24db59af3a67108f81c85 2024-11-07T12:36:32,983 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/21e165e0832e4283a79b62bdb912b779 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/21e165e0832e4283a79b62bdb912b779 2024-11-07T12:36:32,984 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f8ea9d3cfdcb4b50b8f720fb9759d784 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f8ea9d3cfdcb4b50b8f720fb9759d784 2024-11-07T12:36:32,985 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9ff75a9b2863401f9ce79dd6ffab26e7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/9ff75a9b2863401f9ce79dd6ffab26e7 2024-11-07T12:36:32,986 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e8c033ec5c4a4018b86d35d8b9ee834c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/e8c033ec5c4a4018b86d35d8b9ee834c 2024-11-07T12:36:32,987 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f984ef7b142b48e591eb9b7d5062c054 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f984ef7b142b48e591eb9b7d5062c054 2024-11-07T12:36:32,988 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f51663cd20b64ac68758305142055e9f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f51663cd20b64ac68758305142055e9f 2024-11-07T12:36:32,989 DEBUG [StoreCloser-TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/3e19ec07001d496f9077e5055d7ff185 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/3e19ec07001d496f9077e5055d7ff185 2024-11-07T12:36:32,996 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/recovered.edits/566.seqid, newMaxSeqId=566, maxSeqId=1 2024-11-07T12:36:32,997 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. 2024-11-07T12:36:32,997 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for f7b6889d440fde956272a8aab00142f0: Waiting for close lock at 1730982990534Running coprocessor pre-close hooks at 1730982990534Disabling compacts and flushes for region at 1730982990534Disabling writes for close at 1730982991568 (+1034 ms)Obtaining lock to block concurrent updates at 1730982991568Preparing flush snapshotting stores in f7b6889d440fde956272a8aab00142f0 at 1730982991568Finished memstore snapshotting TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1730982991568Flushing stores of TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0. at 1730982991568Flushing f7b6889d440fde956272a8aab00142f0/A: creating writer at 1730982991568Flushing f7b6889d440fde956272a8aab00142f0/A: appending metadata at 1730982991571 (+3 ms)Flushing f7b6889d440fde956272a8aab00142f0/A: closing flushed file at 1730982991571Flushing f7b6889d440fde956272a8aab00142f0/B: creating writer at 1730982991981 (+410 ms)Flushing f7b6889d440fde956272a8aab00142f0/B: appending metadata at 1730982991983 (+2 ms)Flushing f7b6889d440fde956272a8aab00142f0/B: closing flushed file at 1730982991983Flushing f7b6889d440fde956272a8aab00142f0/C: creating writer at 1730982992394 (+411 ms)Flushing f7b6889d440fde956272a8aab00142f0/C: appending metadata at 1730982992397 (+3 ms)Flushing f7b6889d440fde956272a8aab00142f0/C: closing flushed file at 1730982992397Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ae19fb8: reopening flushed file at 1730982992807 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76d62078: reopening flushed file at 1730982992811 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fbd0772: reopening flushed file at 1730982992816 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for f7b6889d440fde956272a8aab00142f0 in 1253ms, sequenceid=563, compaction requested=true at 1730982992821 (+5 ms)Writing region close event to WAL at 1730982992992 (+171 ms)Running coprocessor post-close hooks at 1730982992996 (+4 ms)Closed at 1730982992997 (+1 ms) 2024-11-07T12:36:33,000 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:33,001 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f7b6889d440fde956272a8aab00142f0, regionState=CLOSED 2024-11-07T12:36:33,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:36:33,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T12:36:33,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure f7b6889d440fde956272a8aab00142f0, server=87fb68174c52,38515,1730982961782 in 2.6270 sec 2024-11-07T12:36:33,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-07T12:36:33,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7b6889d440fde956272a8aab00142f0, UNASSIGN in 2.6340 sec 2024-11-07T12:36:33,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-07T12:36:33,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6420 sec 2024-11-07T12:36:33,012 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982993012"}]},"ts":"1730982993012"} 2024-11-07T12:36:33,013 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:36:33,013 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:36:33,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6580 sec 2024-11-07T12:36:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:36:34,537 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:34,541 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T12:36:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:34,547 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:36:34,549 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:34,553 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:34,557 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/recovered.edits] 2024-11-07T12:36:34,559 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/233d9caa0a9b49ecba2e217ee08e85a6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/233d9caa0a9b49ecba2e217ee08e85a6 2024-11-07T12:36:34,561 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55fdf7fb2a964e1b8b08891c61ac34b0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/55fdf7fb2a964e1b8b08891c61ac34b0 2024-11-07T12:36:34,562 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5def71e6b43e42e39692b0d6988cb92c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/A/5def71e6b43e42e39692b0d6988cb92c 2024-11-07T12:36:34,565 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0f152c21a7264e1e9cb1769c491ecbe4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/0f152c21a7264e1e9cb1769c491ecbe4 2024-11-07T12:36:34,566 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a1d6db4460245b687b5f725a7375d9a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/6a1d6db4460245b687b5f725a7375d9a 2024-11-07T12:36:34,568 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/87e3059635c541a286dd7a7b7ea8b6f4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/B/87e3059635c541a286dd7a7b7ea8b6f4 2024-11-07T12:36:34,570 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/84da1b6fa5a14d1f80bb511c384ddde8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/84da1b6fa5a14d1f80bb511c384ddde8 2024-11-07T12:36:34,571 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/85068c2245744c5ea2b6eb51f87a7de1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/85068c2245744c5ea2b6eb51f87a7de1 2024-11-07T12:36:34,573 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f74f6f4c1dd04ab5b979116f789d53c9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/C/f74f6f4c1dd04ab5b979116f789d53c9 2024-11-07T12:36:34,576 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/recovered.edits/566.seqid to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0/recovered.edits/566.seqid 2024-11-07T12:36:34,576 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/f7b6889d440fde956272a8aab00142f0 2024-11-07T12:36:34,576 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:36:34,581 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:34,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-07T12:36:34,590 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:36:34,621 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:36:34,622 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:34,622 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:36:34,623 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730982994622"}]},"ts":"9223372036854775807"} 2024-11-07T12:36:34,627 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:36:34,627 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f7b6889d440fde956272a8aab00142f0, NAME => 'TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:36:34,627 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:36:34,627 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730982994627"}]},"ts":"9223372036854775807"} 2024-11-07T12:36:34,630 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:36:34,631 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:34,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-07T12:36:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:36:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:36:34,658 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:34,670 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;87fb68174c52:38515-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1590488155_22 at /127.0.0.1:37314 [Waiting for operation #191] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1590488155_22 at /127.0.0.1:38484 [Waiting for operation #273] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1848162525_22 at /127.0.0.1:37114 [Waiting for operation #255] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/87fb68174c52:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=465 (was 310) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=6096 (was 6725) 2024-11-07T12:36:34,679 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=465, ProcessCount=11, AvailableMemoryMB=6096 2024-11-07T12:36:34,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:36:34,680 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:36:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:34,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:36:34,683 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:34,683 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-07T12:36:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-07T12:36:34,684 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:36:34,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741954_1130 (size=959) 2024-11-07T12:36:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-07T12:36:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-07T12:36:35,092 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e 2024-11-07T12:36:35,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741955_1131 (size=53) 2024-11-07T12:36:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-07T12:36:35,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:35,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 292f80e0c9fe1beaa809aba371c91f47, disabling compactions & flushes 2024-11-07T12:36:35,501 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:35,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:35,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. after waiting 0 ms 2024-11-07T12:36:35,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:35,501 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:35,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 292f80e0c9fe1beaa809aba371c91f47: Waiting for close lock at 1730982995501Disabling compacts and flushes for region at 1730982995501Disabling writes for close at 1730982995501Writing region close event to WAL at 1730982995501Closed at 1730982995501 2024-11-07T12:36:35,503 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:36:35,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730982995503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982995503"}]},"ts":"1730982995503"} 2024-11-07T12:36:35,505 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:36:35,506 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:36:35,506 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982995506"}]},"ts":"1730982995506"} 2024-11-07T12:36:35,507 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:36:35,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, ASSIGN}] 2024-11-07T12:36:35,509 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, ASSIGN 2024-11-07T12:36:35,509 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, ASSIGN; state=OFFLINE, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=false 2024-11-07T12:36:35,660 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=292f80e0c9fe1beaa809aba371c91f47, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:35,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, ASSIGN because future has completed 2024-11-07T12:36:35,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:36:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-07T12:36:35,818 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:35,818 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:36:35,819 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,819 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:35,819 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,819 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,820 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,822 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 292f80e0c9fe1beaa809aba371c91f47 columnFamilyName A 2024-11-07T12:36:35,822 DEBUG [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:35,823 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(327): Store=292f80e0c9fe1beaa809aba371c91f47/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:35,823 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,824 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 292f80e0c9fe1beaa809aba371c91f47 columnFamilyName B 2024-11-07T12:36:35,824 DEBUG [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:35,825 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(327): Store=292f80e0c9fe1beaa809aba371c91f47/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:35,825 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,826 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 292f80e0c9fe1beaa809aba371c91f47 columnFamilyName C 2024-11-07T12:36:35,826 DEBUG [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:35,826 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(327): Store=292f80e0c9fe1beaa809aba371c91f47/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:35,826 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,827 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,827 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,828 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,828 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,828 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:36:35,829 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,831 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:36:35,832 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 292f80e0c9fe1beaa809aba371c91f47; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75460603, jitterRate=0.12445060908794403}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:36:35,832 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:35,833 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 292f80e0c9fe1beaa809aba371c91f47: Running coprocessor pre-open hook at 1730982995819Writing region info on filesystem at 1730982995819Initializing all the Stores at 1730982995820 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982995820Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982995820Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982995820Cleaning up temporary data from old regions at 1730982995828 (+8 ms)Running coprocessor post-open hooks at 1730982995832 (+4 ms)Region opened successfully at 1730982995832 2024-11-07T12:36:35,833 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., pid=30, masterSystemTime=1730982995814 2024-11-07T12:36:35,835 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:35,835 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:35,836 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=292f80e0c9fe1beaa809aba371c91f47, regionState=OPEN, openSeqNum=2, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:35,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:36:35,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-07T12:36:35,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 in 176 msec 2024-11-07T12:36:35,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-07T12:36:35,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, ASSIGN in 333 msec 2024-11-07T12:36:35,843 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:36:35,844 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982995843"}]},"ts":"1730982995843"} 2024-11-07T12:36:35,845 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:36:35,846 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:36:35,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-07T12:36:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-07T12:36:36,828 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:36,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:36,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:36,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:36,831 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c62a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:36,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5de43e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:36,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:36,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:36,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:36,842 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:36:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb68174c52,36251,1730982961115 2024-11-07T12:36:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d953145 2024-11-07T12:36:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:36:36,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:36:36,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:36:36,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:36:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:36:36,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741956_1132 (size=995) 2024-11-07T12:36:37,273 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T12:36:37,273 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T12:36:37,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:36:37,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, REOPEN/MOVE}] 2024-11-07T12:36:37,284 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, REOPEN/MOVE 2024-11-07T12:36:37,285 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=292f80e0c9fe1beaa809aba371c91f47, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:37,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, REOPEN/MOVE because future has completed 2024-11-07T12:36:37,288 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:36:37,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:36:37,440 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,440 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:36:37,440 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 292f80e0c9fe1beaa809aba371c91f47, disabling compactions & flushes 2024-11-07T12:36:37,440 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:37,440 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:37,440 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. after waiting 0 ms 2024-11-07T12:36:37,440 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:37,444 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:36:37,445 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:37,445 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 292f80e0c9fe1beaa809aba371c91f47: Waiting for close lock at 1730982997440Running coprocessor pre-close hooks at 1730982997440Disabling compacts and flushes for region at 1730982997440Disabling writes for close at 1730982997440Writing region close event to WAL at 1730982997440Running coprocessor post-close hooks at 1730982997445 (+5 ms)Closed at 1730982997445 2024-11-07T12:36:37,445 WARN [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 292f80e0c9fe1beaa809aba371c91f47 to self. 2024-11-07T12:36:37,447 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,447 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=292f80e0c9fe1beaa809aba371c91f47, regionState=CLOSED 2024-11-07T12:36:37,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:36:37,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-07T12:36:37,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 in 162 msec 2024-11-07T12:36:37,453 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, REOPEN/MOVE; state=CLOSED, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=true 2024-11-07T12:36:37,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=292f80e0c9fe1beaa809aba371c91f47, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:37,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, REOPEN/MOVE because future has completed 2024-11-07T12:36:37,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:36:37,762 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:37,762 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:36:37,762 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,763 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:36:37,763 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,763 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,767 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,773 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 292f80e0c9fe1beaa809aba371c91f47 columnFamilyName A 2024-11-07T12:36:37,774 DEBUG [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:37,775 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(327): Store=292f80e0c9fe1beaa809aba371c91f47/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:37,775 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,776 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 292f80e0c9fe1beaa809aba371c91f47 columnFamilyName B 2024-11-07T12:36:37,776 DEBUG [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:37,776 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(327): Store=292f80e0c9fe1beaa809aba371c91f47/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:37,776 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,777 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 292f80e0c9fe1beaa809aba371c91f47 columnFamilyName C 2024-11-07T12:36:37,777 DEBUG [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:37,777 INFO [StoreOpener-292f80e0c9fe1beaa809aba371c91f47-1 {}] regionserver.HStore(327): Store=292f80e0c9fe1beaa809aba371c91f47/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:36:37,777 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,778 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,779 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,780 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,780 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,780 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:36:37,781 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,782 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 292f80e0c9fe1beaa809aba371c91f47; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71049078, jitterRate=0.05871376395225525}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:36:37,782 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,783 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 292f80e0c9fe1beaa809aba371c91f47: Running coprocessor pre-open hook at 1730982997763Writing region info on filesystem at 1730982997763Initializing all the Stores at 1730982997763Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982997764 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982997764Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982997764Cleaning up temporary data from old regions at 1730982997780 (+16 ms)Running coprocessor post-open hooks at 1730982997782 (+2 ms)Region opened successfully at 1730982997783 (+1 ms) 2024-11-07T12:36:37,784 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., pid=35, masterSystemTime=1730982997758 2024-11-07T12:36:37,786 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:37,786 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:37,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=292f80e0c9fe1beaa809aba371c91f47, regionState=OPEN, openSeqNum=5, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:36:37,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:36:37,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-07T12:36:37,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 in 183 msec 2024-11-07T12:36:37,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-07T12:36:37,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, REOPEN/MOVE in 508 msec 2024-11-07T12:36:37,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-07T12:36:37,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-07T12:36:37,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 945 msec 2024-11-07T12:36:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T12:36:37,801 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:37,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,804 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536e6ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,806 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24910c03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,808 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1a2f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,810 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347e25ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,812 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f115a3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,814 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@489930bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,816 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce7dd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,817 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bfdcff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,819 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c0fc52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,821 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec81b5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,824 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@104079ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,825 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fdc2bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,827 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ba7acc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,829 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8f346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,833 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c213e11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@642de49a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:36:37,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:36:37,836 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3d0324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:37,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:36:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393389fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:36:37,839 DEBUG [Thread-667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,840 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,840 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,840 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:37,840 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-07T12:36:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,842 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:36:37,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,844 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:37,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:37,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,846 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,846 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,847 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,852 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,855 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:36:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:36:37,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:36:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:36:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:36:37,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:37,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:36:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d6ae6d7c6d7a4df39357d9314afe6da7_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730982997867/Put/seqid=0 2024-11-07T12:36:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:36:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983057930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983057932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:37,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983057944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:37,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983057946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983057948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:37,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741957_1133 (size=14594) 2024-11-07T12:36:37,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:37,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d6ae6d7c6d7a4df39357d9314afe6da7_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d6ae6d7c6d7a4df39357d9314afe6da7_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:37,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/f076e2dca36c4c238fd7e445818f2afc, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:37,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/f076e2dca36c4c238fd7e445818f2afc is 175, key is test_row_0/A:col10/1730982997867/Put/seqid=0 2024-11-07T12:36:38,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:36:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741958_1134 (size=39549) 2024-11-07T12:36:38,038 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/f076e2dca36c4c238fd7e445818f2afc 2024-11-07T12:36:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983058057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983058057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/8dff7aab42bd4ba1bc6b26679561559e is 50, key is test_row_0/B:col10/1730982997867/Put/seqid=0 2024-11-07T12:36:38,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983058068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983058068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983058068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741959_1135 (size=12001) 2024-11-07T12:36:38,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/8dff7aab42bd4ba1bc6b26679561559e 2024-11-07T12:36:38,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:36:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/9f0e2958110142198d25d8f175c0b292 is 50, key is test_row_0/C:col10/1730982997867/Put/seqid=0 2024-11-07T12:36:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:36:38,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741960_1136 (size=12001) 2024-11-07T12:36:38,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983058268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983058268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983058287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983058287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983058288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:36:38,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:38,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:36:38,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:38,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:36:38,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983058578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,581 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:36:38,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/9f0e2958110142198d25d8f175c0b292 2024-11-07T12:36:38,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983058579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/f076e2dca36c4c238fd7e445818f2afc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f076e2dca36c4c238fd7e445818f2afc 2024-11-07T12:36:38,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983058598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983058598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983058599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:38,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f076e2dca36c4c238fd7e445818f2afc, entries=200, sequenceid=16, filesize=38.6 K 2024-11-07T12:36:38,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:36:38,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:38,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/8dff7aab42bd4ba1bc6b26679561559e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/8dff7aab42bd4ba1bc6b26679561559e 2024-11-07T12:36:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:38,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/8dff7aab42bd4ba1bc6b26679561559e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T12:36:38,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/9f0e2958110142198d25d8f175c0b292 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/9f0e2958110142198d25d8f175c0b292 2024-11-07T12:36:38,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/9f0e2958110142198d25d8f175c0b292, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T12:36:38,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 292f80e0c9fe1beaa809aba371c91f47 in 772ms, sequenceid=16, compaction requested=false 2024-11-07T12:36:38,646 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T12:36:38,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:38,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:36:38,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:38,770 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:36:38,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107888589217a0c4a169a38dd814eba6f8e_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730982997941/Put/seqid=0 2024-11-07T12:36:38,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741961_1137 (size=12154) 2024-11-07T12:36:38,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:38,809 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107888589217a0c4a169a38dd814eba6f8e_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107888589217a0c4a169a38dd814eba6f8e_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:38,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/e843ad3458984638a2d63f8717422d4a, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:38,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/e843ad3458984638a2d63f8717422d4a is 175, key is test_row_0/A:col10/1730982997941/Put/seqid=0 2024-11-07T12:36:38,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741962_1138 (size=30955) 2024-11-07T12:36:38,829 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/e843ad3458984638a2d63f8717422d4a 2024-11-07T12:36:38,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/ae714a52dfec4c03bd9b21e3879b074b is 50, key is test_row_0/B:col10/1730982997941/Put/seqid=0 2024-11-07T12:36:38,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741963_1139 (size=12001) 2024-11-07T12:36:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:36:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983059115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983059117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983059117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983059117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983059120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983059228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983059228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983059230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983059230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,254 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/ae714a52dfec4c03bd9b21e3879b074b 2024-11-07T12:36:39,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/126f288507834ae5b2868a7821c0cce9 is 50, key is test_row_0/C:col10/1730982997941/Put/seqid=0 2024-11-07T12:36:39,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741964_1140 (size=12001) 2024-11-07T12:36:39,273 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/126f288507834ae5b2868a7821c0cce9 2024-11-07T12:36:39,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/e843ad3458984638a2d63f8717422d4a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/e843ad3458984638a2d63f8717422d4a 2024-11-07T12:36:39,290 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/e843ad3458984638a2d63f8717422d4a, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T12:36:39,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/ae714a52dfec4c03bd9b21e3879b074b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/ae714a52dfec4c03bd9b21e3879b074b 2024-11-07T12:36:39,299 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/ae714a52dfec4c03bd9b21e3879b074b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T12:36:39,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/126f288507834ae5b2868a7821c0cce9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/126f288507834ae5b2868a7821c0cce9 2024-11-07T12:36:39,327 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/126f288507834ae5b2868a7821c0cce9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T12:36:39,328 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 292f80e0c9fe1beaa809aba371c91f47 in 558ms, sequenceid=40, compaction requested=false 2024-11-07T12:36:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:39,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-07T12:36:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-07T12:36:39,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T12:36:39,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4870 sec 2024-11-07T12:36:39,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.4940 sec 2024-11-07T12:36:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:39,440 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:39,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:36:39,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110744bb6bde89064e5a85d850c373bbfce7_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730982999117/Put/seqid=0 2024-11-07T12:36:39,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741965_1141 (size=14594) 2024-11-07T12:36:39,488 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:39,495 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110744bb6bde89064e5a85d850c373bbfce7_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110744bb6bde89064e5a85d850c373bbfce7_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:39,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/1c9b49c3d3be4200a5f7825fff0b5550, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:39,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/1c9b49c3d3be4200a5f7825fff0b5550 is 175, key is test_row_0/A:col10/1730982999117/Put/seqid=0 2024-11-07T12:36:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983059489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983059492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983059489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983059492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741966_1142 (size=39549) 2024-11-07T12:36:39,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/1c9b49c3d3be4200a5f7825fff0b5550 2024-11-07T12:36:39,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/556c5c5a78444f79a237b41b3f144988 is 50, key is test_row_0/B:col10/1730982999117/Put/seqid=0 2024-11-07T12:36:39,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741967_1143 (size=12001) 2024-11-07T12:36:39,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/556c5c5a78444f79a237b41b3f144988 2024-11-07T12:36:39,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983059607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/84690929296e4569abe25a4d7d8bd550 is 50, key is test_row_0/C:col10/1730982999117/Put/seqid=0 2024-11-07T12:36:39,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983059608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983059610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983059611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741968_1144 (size=12001) 2024-11-07T12:36:39,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983059817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983059817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983059828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983059828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:36:39,988 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:39,988 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-07T12:36:39,992 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:39,994 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:39,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:36:40,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/84690929296e4569abe25a4d7d8bd550 2024-11-07T12:36:40,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/1c9b49c3d3be4200a5f7825fff0b5550 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/1c9b49c3d3be4200a5f7825fff0b5550 2024-11-07T12:36:40,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/1c9b49c3d3be4200a5f7825fff0b5550, entries=200, sequenceid=53, filesize=38.6 K 2024-11-07T12:36:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/556c5c5a78444f79a237b41b3f144988 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/556c5c5a78444f79a237b41b3f144988 2024-11-07T12:36:40,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/556c5c5a78444f79a237b41b3f144988, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T12:36:40,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/84690929296e4569abe25a4d7d8bd550 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/84690929296e4569abe25a4d7d8bd550 2024-11-07T12:36:40,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/84690929296e4569abe25a4d7d8bd550, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T12:36:40,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 292f80e0c9fe1beaa809aba371c91f47 in 637ms, sequenceid=53, compaction requested=true 2024-11-07T12:36:40,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:40,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:40,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:40,078 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:40,078 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:40,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:40,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:40,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:40,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:40,080 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:40,080 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:40,080 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:40,080 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,080 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:40,080 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,080 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f076e2dca36c4c238fd7e445818f2afc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/e843ad3458984638a2d63f8717422d4a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/1c9b49c3d3be4200a5f7825fff0b5550] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=107.5 K 2024-11-07T12:36:40,080 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,080 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f076e2dca36c4c238fd7e445818f2afc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/e843ad3458984638a2d63f8717422d4a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/1c9b49c3d3be4200a5f7825fff0b5550] 2024-11-07T12:36:40,081 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/8dff7aab42bd4ba1bc6b26679561559e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/ae714a52dfec4c03bd9b21e3879b074b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/556c5c5a78444f79a237b41b3f144988] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=35.2 K 2024-11-07T12:36:40,081 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting f076e2dca36c4c238fd7e445818f2afc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730982997854 2024-11-07T12:36:40,082 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dff7aab42bd4ba1bc6b26679561559e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730982997867 2024-11-07T12:36:40,084 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting ae714a52dfec4c03bd9b21e3879b074b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982997924 2024-11-07T12:36:40,084 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting e843ad3458984638a2d63f8717422d4a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982997924 2024-11-07T12:36:40,084 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 556c5c5a78444f79a237b41b3f144988, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982999107 2024-11-07T12:36:40,084 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9b49c3d3be4200a5f7825fff0b5550, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982999107 2024-11-07T12:36:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:36:40,101 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:40,106 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:40,107 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/48c3e658bc9b448592f714b8587e2862 is 50, key is test_row_0/B:col10/1730982999117/Put/seqid=0 2024-11-07T12:36:40,108 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110789714982540e4e3085093e0b4e5995f7_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:40,115 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110789714982540e4e3085093e0b4e5995f7_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:40,115 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110789714982540e4e3085093e0b4e5995f7_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:40,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:40,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:36:40,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:36:40,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:40,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983060159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983060161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741969_1145 (size=12104) 2024-11-07T12:36:40,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983060171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983060170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983060171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073d399c1ff84344d988c401b8688f0fae_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730982999489/Put/seqid=0 2024-11-07T12:36:40,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741970_1146 (size=4469) 2024-11-07T12:36:40,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741971_1147 (size=17034) 2024-11-07T12:36:40,246 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:40,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073d399c1ff84344d988c401b8688f0fae_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073d399c1ff84344d988c401b8688f0fae_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:40,254 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2fb11a4a06414b27a692ec8551ba865a, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:40,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2fb11a4a06414b27a692ec8551ba865a is 175, key is test_row_0/A:col10/1730982999489/Put/seqid=0 2024-11-07T12:36:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741972_1148 (size=48139) 2024-11-07T12:36:40,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2fb11a4a06414b27a692ec8551ba865a 2024-11-07T12:36:40,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983060278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983060278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/76c0121a0bdd433a8a556da621769ee2 is 50, key is test_row_0/B:col10/1730982999489/Put/seqid=0 2024-11-07T12:36:40,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:36:40,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:40,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983060297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983060298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983060298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:36:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741973_1149 (size=12001) 2024-11-07T12:36:40,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:36:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983060488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983060488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983060517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983060517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983060517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,588 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/48c3e658bc9b448592f714b8587e2862 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/48c3e658bc9b448592f714b8587e2862 2024-11-07T12:36:40,594 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into 48c3e658bc9b448592f714b8587e2862(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:40,594 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:40,594 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=13, startTime=1730983000078; duration=0sec 2024-11-07T12:36:40,595 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:40,595 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:40,595 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:40,598 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:40,598 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:40,598 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,598 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/9f0e2958110142198d25d8f175c0b292, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/126f288507834ae5b2868a7821c0cce9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/84690929296e4569abe25a4d7d8bd550] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=35.2 K 2024-11-07T12:36:40,598 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f0e2958110142198d25d8f175c0b292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730982997867 2024-11-07T12:36:40,599 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 126f288507834ae5b2868a7821c0cce9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982997924 2024-11-07T12:36:40,599 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 84690929296e4569abe25a4d7d8bd550, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982999107 2024-11-07T12:36:40,609 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:40,610 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/dd80b8eade4345e283c82661bc8d638b is 50, key is test_row_0/C:col10/1730982999117/Put/seqid=0 2024-11-07T12:36:40,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741974_1150 (size=12104) 2024-11-07T12:36:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:36:40,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:36:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,627 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#126 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:40,627 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/b8e710a250d443c68f5d4c63f478d7df is 175, key is test_row_0/A:col10/1730982999117/Put/seqid=0 2024-11-07T12:36:40,634 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/dd80b8eade4345e283c82661bc8d638b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/dd80b8eade4345e283c82661bc8d638b 2024-11-07T12:36:40,643 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into dd80b8eade4345e283c82661bc8d638b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:40,643 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:40,643 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=13, startTime=1730983000079; duration=0sec 2024-11-07T12:36:40,643 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:40,643 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:40,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741975_1151 (size=31058) 2024-11-07T12:36:40,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/76c0121a0bdd433a8a556da621769ee2 2024-11-07T12:36:40,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/d31a05a9eda341a8a46dc69952c7a8eb is 50, key is test_row_0/C:col10/1730982999489/Put/seqid=0 2024-11-07T12:36:40,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741976_1152 (size=12001) 2024-11-07T12:36:40,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/d31a05a9eda341a8a46dc69952c7a8eb 2024-11-07T12:36:40,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2fb11a4a06414b27a692ec8551ba865a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2fb11a4a06414b27a692ec8551ba865a 2024-11-07T12:36:40,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:36:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2fb11a4a06414b27a692ec8551ba865a, entries=250, sequenceid=78, filesize=47.0 K 2024-11-07T12:36:40,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/76c0121a0bdd433a8a556da621769ee2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/76c0121a0bdd433a8a556da621769ee2 2024-11-07T12:36:40,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/76c0121a0bdd433a8a556da621769ee2, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T12:36:40,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/d31a05a9eda341a8a46dc69952c7a8eb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/d31a05a9eda341a8a46dc69952c7a8eb 2024-11-07T12:36:40,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/d31a05a9eda341a8a46dc69952c7a8eb, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T12:36:40,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 292f80e0c9fe1beaa809aba371c91f47 in 658ms, sequenceid=78, compaction requested=false 2024-11-07T12:36:40,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:40,804 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:40,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:36:40,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075d45a470fa2444928a29ce5938abbf14_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983000802/Put/seqid=0 2024-11-07T12:36:40,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983060853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983060855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983060857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983060862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983060863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741977_1153 (size=17034) 2024-11-07T12:36:40,874 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:40,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075d45a470fa2444928a29ce5938abbf14_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d45a470fa2444928a29ce5938abbf14_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:40,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/624984436f004d469565af9fa8eac5c3, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:40,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/624984436f004d469565af9fa8eac5c3 is 175, key is test_row_0/A:col10/1730983000802/Put/seqid=0 2024-11-07T12:36:40,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741978_1154 (size=48139) 2024-11-07T12:36:40,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/624984436f004d469565af9fa8eac5c3 2024-11-07T12:36:40,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/b0cbb8b7d4114d2798f6fa74dbd1d13d is 50, key is test_row_0/B:col10/1730983000802/Put/seqid=0 2024-11-07T12:36:40,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741979_1155 (size=12001) 2024-11-07T12:36:40,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/b0cbb8b7d4114d2798f6fa74dbd1d13d 2024-11-07T12:36:40,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:36:40,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:40,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:40,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:40,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/702df6069cc549e8b77eb3bd43ce5c5c is 50, key is test_row_0/C:col10/1730983000802/Put/seqid=0 2024-11-07T12:36:40,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741980_1156 (size=12001) 2024-11-07T12:36:40,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/702df6069cc549e8b77eb3bd43ce5c5c 2024-11-07T12:36:40,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983060967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983060967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983060967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/624984436f004d469565af9fa8eac5c3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/624984436f004d469565af9fa8eac5c3 2024-11-07T12:36:40,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/624984436f004d469565af9fa8eac5c3, entries=250, sequenceid=93, filesize=47.0 K 2024-11-07T12:36:40,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983060977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983060978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:40,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/b0cbb8b7d4114d2798f6fa74dbd1d13d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b0cbb8b7d4114d2798f6fa74dbd1d13d 2024-11-07T12:36:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:40,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b0cbb8b7d4114d2798f6fa74dbd1d13d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T12:36:40,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/702df6069cc549e8b77eb3bd43ce5c5c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/702df6069cc549e8b77eb3bd43ce5c5c 2024-11-07T12:36:41,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/702df6069cc549e8b77eb3bd43ce5c5c, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T12:36:41,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 292f80e0c9fe1beaa809aba371c91f47 in 203ms, sequenceid=93, compaction requested=true 2024-11-07T12:36:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:36:41,007 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-07T12:36:41,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:36:41,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:36:41,009 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. because compaction request was cancelled 2024-11-07T12:36:41,009 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:41,009 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:41,013 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:41,013 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:41,013 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:41,013 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/48c3e658bc9b448592f714b8587e2862, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/76c0121a0bdd433a8a556da621769ee2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b0cbb8b7d4114d2798f6fa74dbd1d13d] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=35.3 K 2024-11-07T12:36:41,014 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 48c3e658bc9b448592f714b8587e2862, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982999107 2024-11-07T12:36:41,014 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c0121a0bdd433a8a556da621769ee2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730982999489 2024-11-07T12:36:41,014 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b0cbb8b7d4114d2798f6fa74dbd1d13d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730983000164 2024-11-07T12:36:41,036 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#135 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:41,037 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/044e5356cc4242d3a92c28bde9510141 is 50, key is test_row_0/B:col10/1730983000802/Put/seqid=0 2024-11-07T12:36:41,066 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/b8e710a250d443c68f5d4c63f478d7df as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b8e710a250d443c68f5d4c63f478d7df 2024-11-07T12:36:41,074 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into b8e710a250d443c68f5d4c63f478d7df(size=30.3 K), total size for store is 124.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:41,074 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:41,074 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=13, startTime=1730983000078; duration=0sec 2024-11-07T12:36:41,074 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:41,074 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:41,075 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:41,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741981_1157 (size=12207) 2024-11-07T12:36:41,080 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:41,081 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:41,082 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:41,082 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/044e5356cc4242d3a92c28bde9510141 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/044e5356cc4242d3a92c28bde9510141 2024-11-07T12:36:41,083 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/dd80b8eade4345e283c82661bc8d638b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/d31a05a9eda341a8a46dc69952c7a8eb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/702df6069cc549e8b77eb3bd43ce5c5c] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=35.3 K 2024-11-07T12:36:41,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:36:41,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:41,085 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:36:41,085 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd80b8eade4345e283c82661bc8d638b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982999107 2024-11-07T12:36:41,087 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting d31a05a9eda341a8a46dc69952c7a8eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730982999489 2024-11-07T12:36:41,087 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702df6069cc549e8b77eb3bd43ce5c5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730983000164 2024-11-07T12:36:41,093 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into 044e5356cc4242d3a92c28bde9510141(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:41,093 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:41,093 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=13, startTime=1730983001007; duration=0sec 2024-11-07T12:36:41,093 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:41,093 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:41,106 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:41,107 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/f3fa9151b5884a0e8f4481cceb6772fb is 50, key is test_row_0/C:col10/1730983000802/Put/seqid=0 2024-11-07T12:36:41,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107022e13e586fa4664a298adba96e3744d_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983000861/Put/seqid=0 2024-11-07T12:36:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:36:41,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741982_1158 (size=12207) 2024-11-07T12:36:41,147 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/f3fa9151b5884a0e8f4481cceb6772fb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f3fa9151b5884a0e8f4481cceb6772fb 2024-11-07T12:36:41,153 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into f3fa9151b5884a0e8f4481cceb6772fb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:41,153 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:41,153 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=13, startTime=1730983001007; duration=0sec 2024-11-07T12:36:41,154 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:41,154 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741983_1159 (size=12154) 2024-11-07T12:36:41,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:41,165 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107022e13e586fa4664a298adba96e3744d_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107022e13e586fa4664a298adba96e3744d_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:41,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2b84fc0e6d384d70991b8bf7da53c620, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:41,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2b84fc0e6d384d70991b8bf7da53c620 is 175, key is test_row_0/A:col10/1730983000861/Put/seqid=0 2024-11-07T12:36:41,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:41,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741984_1160 (size=30955) 2024-11-07T12:36:41,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983061207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983061208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983061208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983061213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983061213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983061317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983061318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983061320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983061327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983061327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983061528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983061528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983061528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983061537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983061537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,600 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2b84fc0e6d384d70991b8bf7da53c620 2024-11-07T12:36:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/f7fdabed82704a4aa2e18b35ffbe20d4 is 50, key is test_row_0/B:col10/1730983000861/Put/seqid=0 2024-11-07T12:36:41,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741985_1161 (size=12001) 2024-11-07T12:36:41,653 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/f7fdabed82704a4aa2e18b35ffbe20d4 2024-11-07T12:36:41,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/b36831cc46994d84a9b7defd27bf97a5 is 50, key is test_row_0/C:col10/1730983000861/Put/seqid=0 2024-11-07T12:36:41,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741986_1162 (size=12001) 2024-11-07T12:36:41,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983061837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983061837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983061838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983061848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983061848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,079 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/b36831cc46994d84a9b7defd27bf97a5 2024-11-07T12:36:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2b84fc0e6d384d70991b8bf7da53c620 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2b84fc0e6d384d70991b8bf7da53c620 2024-11-07T12:36:42,100 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2b84fc0e6d384d70991b8bf7da53c620, entries=150, sequenceid=118, filesize=30.2 K 2024-11-07T12:36:42,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/f7fdabed82704a4aa2e18b35ffbe20d4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f7fdabed82704a4aa2e18b35ffbe20d4 2024-11-07T12:36:42,119 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f7fdabed82704a4aa2e18b35ffbe20d4, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T12:36:42,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/b36831cc46994d84a9b7defd27bf97a5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/b36831cc46994d84a9b7defd27bf97a5 2024-11-07T12:36:42,128 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/b36831cc46994d84a9b7defd27bf97a5, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T12:36:42,131 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 292f80e0c9fe1beaa809aba371c91f47 in 1046ms, sequenceid=118, compaction requested=true 2024-11-07T12:36:42,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:42,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:42,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-07T12:36:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-07T12:36:42,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-07T12:36:42,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1390 sec 2024-11-07T12:36:42,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.1510 sec 2024-11-07T12:36:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:36:42,148 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:42,148 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-07T12:36:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:36:42,153 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:42,154 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:42,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:36:42,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:36:42,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:42,308 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:36:42,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079b55f13d51b34aea9def067747a33197_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983001194/Put/seqid=0 2024-11-07T12:36:42,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741987_1163 (size=12254) 2024-11-07T12:36:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:42,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:42,352 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079b55f13d51b34aea9def067747a33197_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079b55f13d51b34aea9def067747a33197_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/c31d7a5a4a2945a99763e4ef9b0e9e8b, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/c31d7a5a4a2945a99763e4ef9b0e9e8b is 175, key is test_row_0/A:col10/1730983001194/Put/seqid=0 2024-11-07T12:36:42,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741988_1164 (size=31055) 2024-11-07T12:36:42,372 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/c31d7a5a4a2945a99763e4ef9b0e9e8b 2024-11-07T12:36:42,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/e7febe3adc2a4d0da6816022933772c8 is 50, key is test_row_0/B:col10/1730983001194/Put/seqid=0 2024-11-07T12:36:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983062377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983062378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983062379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983062380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983062385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741989_1165 (size=12101) 2024-11-07T12:36:42,395 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/e7febe3adc2a4d0da6816022933772c8 2024-11-07T12:36:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/19d7106242804e32b9254264abe48421 is 50, key is test_row_0/C:col10/1730983001194/Put/seqid=0 2024-11-07T12:36:42,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741990_1166 (size=12101) 2024-11-07T12:36:42,418 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/19d7106242804e32b9254264abe48421 2024-11-07T12:36:42,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/c31d7a5a4a2945a99763e4ef9b0e9e8b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/c31d7a5a4a2945a99763e4ef9b0e9e8b 2024-11-07T12:36:42,434 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/c31d7a5a4a2945a99763e4ef9b0e9e8b, entries=150, sequenceid=132, filesize=30.3 K 2024-11-07T12:36:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/e7febe3adc2a4d0da6816022933772c8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/e7febe3adc2a4d0da6816022933772c8 2024-11-07T12:36:42,446 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/e7febe3adc2a4d0da6816022933772c8, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T12:36:42,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/19d7106242804e32b9254264abe48421 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19d7106242804e32b9254264abe48421 2024-11-07T12:36:42,466 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19d7106242804e32b9254264abe48421, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T12:36:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:36:42,468 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 292f80e0c9fe1beaa809aba371c91f47 in 160ms, sequenceid=132, compaction requested=true 2024-11-07T12:36:42,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:42,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:42,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-07T12:36:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-07T12:36:42,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T12:36:42,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 316 msec 2024-11-07T12:36:42,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 325 msec 2024-11-07T12:36:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:42,500 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:42,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:36:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983062506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983062509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983062509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983062511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983062511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075dcd91c03b66477fa05f3d061d0c5b0e_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983002500/Put/seqid=0 2024-11-07T12:36:42,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741991_1167 (size=19774) 2024-11-07T12:36:42,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983062618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983062628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983062628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983062628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983062629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:36:42,778 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:42,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-07T12:36:42,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:42,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:42,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:36:42,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983062827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983062838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983062838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983062838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983062838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:36:42,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:42,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:42,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:42,974 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075dcd91c03b66477fa05f3d061d0c5b0e_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075dcd91c03b66477fa05f3d061d0c5b0e_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:42,975 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/82c87bd11ab145c496e48e80c3b0f0c8, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:42,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/82c87bd11ab145c496e48e80c3b0f0c8 is 175, key is test_row_0/A:col10/1730983002500/Put/seqid=0 2024-11-07T12:36:42,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741992_1168 (size=57033) 2024-11-07T12:36:43,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:36:43,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983063137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983063148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983063150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983063150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983063150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:43,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:43,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:43,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,396 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/82c87bd11ab145c496e48e80c3b0f0c8 2024-11-07T12:36:43,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/7fd23ce7de5a490087dc6acb2f3e6e03 is 50, key is test_row_0/B:col10/1730983002500/Put/seqid=0 2024-11-07T12:36:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:36:43,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741993_1169 (size=12151) 2024-11-07T12:36:43,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983063657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983063657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983063658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983063658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983063658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:43,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/7fd23ce7de5a490087dc6acb2f3e6e03 2024-11-07T12:36:43,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/65fd4181d6db4f26bc118d510d5403de is 50, key is test_row_0/C:col10/1730983002500/Put/seqid=0 2024-11-07T12:36:43,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741994_1170 (size=12151) 2024-11-07T12:36:43,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:43,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:43,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:43,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:36:44,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:44,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:44,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:44,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:44,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/65fd4181d6db4f26bc118d510d5403de 2024-11-07T12:36:44,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/82c87bd11ab145c496e48e80c3b0f0c8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/82c87bd11ab145c496e48e80c3b0f0c8 2024-11-07T12:36:44,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/82c87bd11ab145c496e48e80c3b0f0c8, entries=300, sequenceid=159, filesize=55.7 K 2024-11-07T12:36:44,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/7fd23ce7de5a490087dc6acb2f3e6e03 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7fd23ce7de5a490087dc6acb2f3e6e03 2024-11-07T12:36:44,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7fd23ce7de5a490087dc6acb2f3e6e03, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T12:36:44,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/65fd4181d6db4f26bc118d510d5403de as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65fd4181d6db4f26bc118d510d5403de 2024-11-07T12:36:44,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65fd4181d6db4f26bc118d510d5403de, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T12:36:44,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 292f80e0c9fe1beaa809aba371c91f47 in 1793ms, sequenceid=159, compaction requested=true 2024-11-07T12:36:44,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:44,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:44,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:44,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:44,293 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-07T12:36:44,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:44,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:44,293 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:44,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:44,297 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:44,297 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:44,297 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,297 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/044e5356cc4242d3a92c28bde9510141, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f7fdabed82704a4aa2e18b35ffbe20d4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/e7febe3adc2a4d0da6816022933772c8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7fd23ce7de5a490087dc6acb2f3e6e03] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=47.3 K 2024-11-07T12:36:44,297 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 246379 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-07T12:36:44,297 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 044e5356cc4242d3a92c28bde9510141, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730983000164 2024-11-07T12:36:44,297 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:44,297 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,298 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b8e710a250d443c68f5d4c63f478d7df, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2fb11a4a06414b27a692ec8551ba865a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/624984436f004d469565af9fa8eac5c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2b84fc0e6d384d70991b8bf7da53c620, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/c31d7a5a4a2945a99763e4ef9b0e9e8b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/82c87bd11ab145c496e48e80c3b0f0c8] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=240.6 K 2024-11-07T12:36:44,298 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,298 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b8e710a250d443c68f5d4c63f478d7df, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2fb11a4a06414b27a692ec8551ba865a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/624984436f004d469565af9fa8eac5c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2b84fc0e6d384d70991b8bf7da53c620, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/c31d7a5a4a2945a99763e4ef9b0e9e8b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/82c87bd11ab145c496e48e80c3b0f0c8] 2024-11-07T12:36:44,298 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f7fdabed82704a4aa2e18b35ffbe20d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730983000843 2024-11-07T12:36:44,299 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8e710a250d443c68f5d4c63f478d7df, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982999107 2024-11-07T12:36:44,299 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb11a4a06414b27a692ec8551ba865a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730982999466 2024-11-07T12:36:44,299 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624984436f004d469565af9fa8eac5c3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730983000145 2024-11-07T12:36:44,299 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e7febe3adc2a4d0da6816022933772c8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983001187 2024-11-07T12:36:44,300 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b84fc0e6d384d70991b8bf7da53c620, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730983000843 2024-11-07T12:36:44,300 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd23ce7de5a490087dc6acb2f3e6e03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730983002383 2024-11-07T12:36:44,300 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31d7a5a4a2945a99763e4ef9b0e9e8b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983001187 2024-11-07T12:36:44,301 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82c87bd11ab145c496e48e80c3b0f0c8, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730983002369 2024-11-07T12:36:44,313 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:44,314 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/dbb0b5fae8174b82882536f7130d46e0 is 50, key is test_row_0/B:col10/1730983002500/Put/seqid=0 2024-11-07T12:36:44,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:36:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,320 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T12:36:44,328 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:44,332 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107698de8c5760c4e4bacde3fa7c807c853_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:44,336 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107698de8c5760c4e4bacde3fa7c807c853_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:44,337 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107698de8c5760c4e4bacde3fa7c807c853_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:44,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741995_1171 (size=12493) 2024-11-07T12:36:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075250eb1a93784b7b852909267b5b7ee9_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983002506/Put/seqid=0 2024-11-07T12:36:44,358 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/dbb0b5fae8174b82882536f7130d46e0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/dbb0b5fae8174b82882536f7130d46e0 2024-11-07T12:36:44,365 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into dbb0b5fae8174b82882536f7130d46e0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:44,366 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:44,366 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=12, startTime=1730983004293; duration=0sec 2024-11-07T12:36:44,366 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:44,366 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:44,366 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:44,368 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:44,368 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:44,368 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,368 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f3fa9151b5884a0e8f4481cceb6772fb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/b36831cc46994d84a9b7defd27bf97a5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19d7106242804e32b9254264abe48421, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65fd4181d6db4f26bc118d510d5403de] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=47.3 K 2024-11-07T12:36:44,369 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f3fa9151b5884a0e8f4481cceb6772fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730983000164 2024-11-07T12:36:44,373 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b36831cc46994d84a9b7defd27bf97a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730983000843 2024-11-07T12:36:44,373 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d7106242804e32b9254264abe48421, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983001187 2024-11-07T12:36:44,374 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fd4181d6db4f26bc118d510d5403de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730983002383 2024-11-07T12:36:44,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741997_1173 (size=12304) 2024-11-07T12:36:44,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741996_1172 (size=4469) 2024-11-07T12:36:44,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:44,403 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#147 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:44,404 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/5fe17f06d1144a7b98c7f8d74486e333 is 175, key is test_row_0/A:col10/1730983002500/Put/seqid=0 2024-11-07T12:36:44,409 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075250eb1a93784b7b852909267b5b7ee9_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075250eb1a93784b7b852909267b5b7ee9_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:44,409 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:44,410 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a9910a2748b34fa78cc372a0644eed88 is 50, key is test_row_0/C:col10/1730983002500/Put/seqid=0 2024-11-07T12:36:44,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/88f9a59f71c84171b3f75a4a599ebab1, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:44,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/88f9a59f71c84171b3f75a4a599ebab1 is 175, key is test_row_0/A:col10/1730983002506/Put/seqid=0 2024-11-07T12:36:44,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741998_1174 (size=31413) 2024-11-07T12:36:44,437 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/5fe17f06d1144a7b98c7f8d74486e333 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/5fe17f06d1144a7b98c7f8d74486e333 2024-11-07T12:36:44,443 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into 5fe17f06d1144a7b98c7f8d74486e333(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:44,443 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:44,443 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=10, startTime=1730983004293; duration=0sec 2024-11-07T12:36:44,444 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:44,444 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:44,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741999_1175 (size=12493) 2024-11-07T12:36:44,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742000_1176 (size=31105) 2024-11-07T12:36:44,469 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/88f9a59f71c84171b3f75a4a599ebab1 2024-11-07T12:36:44,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/7dc34ee739b94f64b7b4eb412f53e02e is 50, key is test_row_0/B:col10/1730983002506/Put/seqid=0 2024-11-07T12:36:44,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742001_1177 (size=12151) 2024-11-07T12:36:44,497 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/7dc34ee739b94f64b7b4eb412f53e02e 2024-11-07T12:36:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/4a5e889597bf4404a3907b7243eb67e4 is 50, key is test_row_0/C:col10/1730983002506/Put/seqid=0 2024-11-07T12:36:44,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742002_1178 (size=12151) 2024-11-07T12:36:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:44,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983064698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983064698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983064700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983064702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983064704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983064807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983064808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983064808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983064808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983064817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:44,858 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a9910a2748b34fa78cc372a0644eed88 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a9910a2748b34fa78cc372a0644eed88 2024-11-07T12:36:44,864 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into a9910a2748b34fa78cc372a0644eed88(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:44,864 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:44,864 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=12, startTime=1730983004293; duration=0sec 2024-11-07T12:36:44,864 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:44,864 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:44,927 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/4a5e889597bf4404a3907b7243eb67e4 2024-11-07T12:36:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:36:44,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/88f9a59f71c84171b3f75a4a599ebab1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/88f9a59f71c84171b3f75a4a599ebab1 2024-11-07T12:36:44,937 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/88f9a59f71c84171b3f75a4a599ebab1, entries=150, sequenceid=168, filesize=30.4 K 2024-11-07T12:36:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/7dc34ee739b94f64b7b4eb412f53e02e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7dc34ee739b94f64b7b4eb412f53e02e 2024-11-07T12:36:44,943 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7dc34ee739b94f64b7b4eb412f53e02e, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T12:36:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/4a5e889597bf4404a3907b7243eb67e4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/4a5e889597bf4404a3907b7243eb67e4 2024-11-07T12:36:44,949 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/4a5e889597bf4404a3907b7243eb67e4, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T12:36:44,950 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 292f80e0c9fe1beaa809aba371c91f47 in 630ms, sequenceid=168, compaction requested=false 2024-11-07T12:36:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-07T12:36:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-07T12:36:44,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T12:36:44,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-07T12:36:44,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.1750 sec 2024-11-07T12:36:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:45,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T12:36:45,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983065025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983065027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983065028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983065028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983065029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077b2114f985e44e3085309adbdd046dd0_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983005021/Put/seqid=0 2024-11-07T12:36:45,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742003_1179 (size=19774) 2024-11-07T12:36:45,055 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077b2114f985e44e3085309adbdd046dd0_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077b2114f985e44e3085309adbdd046dd0_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:45,064 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/153e9c2b3fb0434b877688978000edc1, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:45,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/153e9c2b3fb0434b877688978000edc1 is 175, key is test_row_0/A:col10/1730983005021/Put/seqid=0 2024-11-07T12:36:45,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742004_1180 (size=57033) 2024-11-07T12:36:45,070 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/153e9c2b3fb0434b877688978000edc1 2024-11-07T12:36:45,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/09220d81f3fc4b549eb08997fc8700c1 is 50, key is test_row_0/B:col10/1730983005021/Put/seqid=0 2024-11-07T12:36:45,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742005_1181 (size=12151) 2024-11-07T12:36:45,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/09220d81f3fc4b549eb08997fc8700c1 2024-11-07T12:36:45,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/65506d4a33dd4de0ba21ee1145ae7ad5 is 50, key is test_row_0/C:col10/1730983005021/Put/seqid=0 2024-11-07T12:36:45,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742006_1182 (size=12151) 2024-11-07T12:36:45,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/65506d4a33dd4de0ba21ee1145ae7ad5 2024-11-07T12:36:45,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/153e9c2b3fb0434b877688978000edc1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/153e9c2b3fb0434b877688978000edc1 2024-11-07T12:36:45,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/153e9c2b3fb0434b877688978000edc1, entries=300, sequenceid=201, filesize=55.7 K 2024-11-07T12:36:45,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/09220d81f3fc4b549eb08997fc8700c1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/09220d81f3fc4b549eb08997fc8700c1 2024-11-07T12:36:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/09220d81f3fc4b549eb08997fc8700c1, entries=150, sequenceid=201, filesize=11.9 K 2024-11-07T12:36:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/65506d4a33dd4de0ba21ee1145ae7ad5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65506d4a33dd4de0ba21ee1145ae7ad5 2024-11-07T12:36:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65506d4a33dd4de0ba21ee1145ae7ad5, entries=150, sequenceid=201, filesize=11.9 K 2024-11-07T12:36:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 292f80e0c9fe1beaa809aba371c91f47 in 154ms, sequenceid=201, compaction requested=true 2024-11-07T12:36:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:45,175 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:45,175 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,177 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:45,177 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:45,177 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:45,177 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:45,177 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:45,177 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/5fe17f06d1144a7b98c7f8d74486e333, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/88f9a59f71c84171b3f75a4a599ebab1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/153e9c2b3fb0434b877688978000edc1] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=116.7 K 2024-11-07T12:36:45,177 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:45,177 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:45,177 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/dbb0b5fae8174b82882536f7130d46e0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7dc34ee739b94f64b7b4eb412f53e02e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/09220d81f3fc4b549eb08997fc8700c1] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=35.9 K 2024-11-07T12:36:45,177 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/5fe17f06d1144a7b98c7f8d74486e333, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/88f9a59f71c84171b3f75a4a599ebab1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/153e9c2b3fb0434b877688978000edc1] 2024-11-07T12:36:45,177 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb0b5fae8174b82882536f7130d46e0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730983002383 2024-11-07T12:36:45,178 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fe17f06d1144a7b98c7f8d74486e333, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730983002383 2024-11-07T12:36:45,178 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dc34ee739b94f64b7b4eb412f53e02e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730983002505 2024-11-07T12:36:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,178 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 09220d81f3fc4b549eb08997fc8700c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1730983004702 2024-11-07T12:36:45,178 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f9a59f71c84171b3f75a4a599ebab1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730983002505 2024-11-07T12:36:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,179 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 153e9c2b3fb0434b877688978000edc1, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1730983004695 2024-11-07T12:36:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,197 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:45,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:45,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:36:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,219 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#156 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:45,219 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/b5e2519af5204723a8c32ceb291b8ef1 is 50, key is test_row_0/B:col10/1730983005021/Put/seqid=0 2024-11-07T12:36:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,225 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110745de7d58aa2d4004b550c99af8857445_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,233 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110745de7d58aa2d4004b550c99af8857445_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:45,233 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110745de7d58aa2d4004b550c99af8857445_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110761dcc37ab9594a86ad945cfe0cc7f549_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983005184/Put/seqid=0 2024-11-07T12:36:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742007_1183 (size=12595) 2024-11-07T12:36:45,276 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/b5e2519af5204723a8c32ceb291b8ef1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b5e2519af5204723a8c32ceb291b8ef1 2024-11-07T12:36:45,285 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into b5e2519af5204723a8c32ceb291b8ef1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:45,285 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:45,285 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=13, startTime=1730983005175; duration=0sec 2024-11-07T12:36:45,285 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:45,285 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:45,285 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:45,287 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:45,287 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:45,287 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:45,288 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a9910a2748b34fa78cc372a0644eed88, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/4a5e889597bf4404a3907b7243eb67e4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65506d4a33dd4de0ba21ee1145ae7ad5] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=35.9 K 2024-11-07T12:36:45,288 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a9910a2748b34fa78cc372a0644eed88, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730983002383 2024-11-07T12:36:45,289 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a5e889597bf4404a3907b7243eb67e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730983002505 2024-11-07T12:36:45,289 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 65506d4a33dd4de0ba21ee1145ae7ad5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1730983004702 2024-11-07T12:36:45,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742008_1184 (size=4469) 2024-11-07T12:36:45,295 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#155 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:45,296 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/374ee9574b05482dbabb5b97424d041c is 175, key is test_row_0/A:col10/1730983005021/Put/seqid=0 2024-11-07T12:36:45,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742009_1185 (size=19774) 2024-11-07T12:36:45,310 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:45,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110761dcc37ab9594a86ad945cfe0cc7f549_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110761dcc37ab9594a86ad945cfe0cc7f549_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:45,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/a8ea5296cd8e4acfb12f6aa21115a168, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:45,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/a8ea5296cd8e4acfb12f6aa21115a168 is 175, key is test_row_0/A:col10/1730983005184/Put/seqid=0 2024-11-07T12:36:45,322 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#158 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:45,322 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/caba2b6d0e95405d8bbd4f37cef34831 is 50, key is test_row_0/C:col10/1730983005021/Put/seqid=0 2024-11-07T12:36:45,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742010_1186 (size=31515) 2024-11-07T12:36:45,352 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/374ee9574b05482dbabb5b97424d041c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/374ee9574b05482dbabb5b97424d041c 2024-11-07T12:36:45,361 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into 374ee9574b05482dbabb5b97424d041c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:45,361 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:45,361 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=13, startTime=1730983005175; duration=0sec 2024-11-07T12:36:45,361 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:45,362 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:45,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742012_1188 (size=12595) 2024-11-07T12:36:45,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742011_1187 (size=57033) 2024-11-07T12:36:45,375 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/a8ea5296cd8e4acfb12f6aa21115a168 2024-11-07T12:36:45,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/bc14167aabed415d97a867aed6a930fc is 50, key is test_row_0/B:col10/1730983005184/Put/seqid=0 2024-11-07T12:36:45,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742013_1189 (size=12151) 2024-11-07T12:36:45,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983065404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983065405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983065407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983065407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983065409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,494 INFO [master/87fb68174c52:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T12:36:45,494 INFO [master/87fb68174c52:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T12:36:45,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983065518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983065518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983065518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983065518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983065528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983065727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983065727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983065727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983065737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983065737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:45,759 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/caba2b6d0e95405d8bbd4f37cef34831 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/caba2b6d0e95405d8bbd4f37cef34831 2024-11-07T12:36:45,765 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into caba2b6d0e95405d8bbd4f37cef34831(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:45,765 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:45,765 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=13, startTime=1730983005175; duration=0sec 2024-11-07T12:36:45,765 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:45,765 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:45,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/bc14167aabed415d97a867aed6a930fc 2024-11-07T12:36:45,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/f0d511650640424dae9df279dcbbab6e is 50, key is test_row_0/C:col10/1730983005184/Put/seqid=0 2024-11-07T12:36:45,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742014_1190 (size=12151) 2024-11-07T12:36:45,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/f0d511650640424dae9df279dcbbab6e 2024-11-07T12:36:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/a8ea5296cd8e4acfb12f6aa21115a168 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/a8ea5296cd8e4acfb12f6aa21115a168 2024-11-07T12:36:45,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/a8ea5296cd8e4acfb12f6aa21115a168, entries=300, sequenceid=212, filesize=55.7 K 2024-11-07T12:36:45,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/bc14167aabed415d97a867aed6a930fc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc14167aabed415d97a867aed6a930fc 2024-11-07T12:36:45,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc14167aabed415d97a867aed6a930fc, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T12:36:45,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/f0d511650640424dae9df279dcbbab6e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f0d511650640424dae9df279dcbbab6e 2024-11-07T12:36:45,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f0d511650640424dae9df279dcbbab6e, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T12:36:45,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 292f80e0c9fe1beaa809aba371c91f47 in 667ms, sequenceid=212, compaction requested=false 2024-11-07T12:36:45,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:46,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T12:36:46,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983066049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983066051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983066057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983066061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983066062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107271a3ae111b448afab93b17497f96ea8_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983005404/Put/seqid=0 2024-11-07T12:36:46,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742015_1191 (size=19774) 2024-11-07T12:36:46,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:46,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107271a3ae111b448afab93b17497f96ea8_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107271a3ae111b448afab93b17497f96ea8_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:46,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/092fb86b262547e68c9de852fb17401f, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:46,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/092fb86b262547e68c9de852fb17401f is 175, key is test_row_0/A:col10/1730983005404/Put/seqid=0 2024-11-07T12:36:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983066167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983066167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983066167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742016_1192 (size=57033) 2024-11-07T12:36:46,181 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/092fb86b262547e68c9de852fb17401f 2024-11-07T12:36:46,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983066178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983066178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/d1d7be02dce941cd903a293a28a19d89 is 50, key is test_row_0/B:col10/1730983005404/Put/seqid=0 2024-11-07T12:36:46,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742017_1193 (size=12151) 2024-11-07T12:36:46,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/d1d7be02dce941cd903a293a28a19d89 2024-11-07T12:36:46,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/fafafac1b8af4388b5e9e900f7c00e0f is 50, key is test_row_0/C:col10/1730983005404/Put/seqid=0 2024-11-07T12:36:46,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742018_1194 (size=12151) 2024-11-07T12:36:46,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983066377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983066377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983066377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983066387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983066388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/fafafac1b8af4388b5e9e900f7c00e0f 2024-11-07T12:36:46,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/092fb86b262547e68c9de852fb17401f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/092fb86b262547e68c9de852fb17401f 2024-11-07T12:36:46,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/092fb86b262547e68c9de852fb17401f, entries=300, sequenceid=243, filesize=55.7 K 2024-11-07T12:36:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/d1d7be02dce941cd903a293a28a19d89 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/d1d7be02dce941cd903a293a28a19d89 2024-11-07T12:36:46,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/d1d7be02dce941cd903a293a28a19d89, entries=150, sequenceid=243, filesize=11.9 K 2024-11-07T12:36:46,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/fafafac1b8af4388b5e9e900f7c00e0f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/fafafac1b8af4388b5e9e900f7c00e0f 2024-11-07T12:36:46,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/fafafac1b8af4388b5e9e900f7c00e0f, entries=150, sequenceid=243, filesize=11.9 K 2024-11-07T12:36:46,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 292f80e0c9fe1beaa809aba371c91f47 in 608ms, sequenceid=243, compaction requested=true 2024-11-07T12:36:46,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:46,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:46,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:46,650 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:46,650 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:46,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:46,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:46,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:46,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:46,651 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 145581 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:46,651 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:46,651 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:46,651 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:46,651 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:46,651 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:46,651 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/374ee9574b05482dbabb5b97424d041c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/a8ea5296cd8e4acfb12f6aa21115a168, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/092fb86b262547e68c9de852fb17401f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=142.2 K 2024-11-07T12:36:46,651 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b5e2519af5204723a8c32ceb291b8ef1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc14167aabed415d97a867aed6a930fc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/d1d7be02dce941cd903a293a28a19d89] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=36.0 K 2024-11-07T12:36:46,651 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:46,651 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/374ee9574b05482dbabb5b97424d041c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/a8ea5296cd8e4acfb12f6aa21115a168, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/092fb86b262547e68c9de852fb17401f] 2024-11-07T12:36:46,652 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e2519af5204723a8c32ceb291b8ef1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1730983004702 2024-11-07T12:36:46,652 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 374ee9574b05482dbabb5b97424d041c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1730983004702 2024-11-07T12:36:46,652 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bc14167aabed415d97a867aed6a930fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983005184 2024-11-07T12:36:46,653 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ea5296cd8e4acfb12f6aa21115a168, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983005026 2024-11-07T12:36:46,653 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d7be02dce941cd903a293a28a19d89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730983005404 2024-11-07T12:36:46,653 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 092fb86b262547e68c9de852fb17401f, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730983005387 2024-11-07T12:36:46,668 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:46,670 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:46,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/c4e03ff14a5e4a5493ab9f7e292055e7 is 50, key is test_row_0/B:col10/1730983005404/Put/seqid=0 2024-11-07T12:36:46,672 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110738d380e92639430faac0b89aaa7e758c_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:46,675 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110738d380e92639430faac0b89aaa7e758c_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:46,675 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110738d380e92639430faac0b89aaa7e758c_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:46,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742019_1195 (size=12697) 2024-11-07T12:36:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:46,691 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:46,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:36:46,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742020_1196 (size=4469) 2024-11-07T12:36:46,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075807873811c249e0a864a2dcafb183b2_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983006691/Put/seqid=0 2024-11-07T12:36:46,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983066725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983066726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983066727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983066728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742021_1197 (size=14794) 2024-11-07T12:36:46,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983066730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983066837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983066837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983066837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983066848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983066848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:36:46,948 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:46,949 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-07T12:36:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:36:46,951 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:46,952 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:46,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983067047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983067047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983067048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:36:47,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983067058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983067058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,091 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/c4e03ff14a5e4a5493ab9f7e292055e7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c4e03ff14a5e4a5493ab9f7e292055e7 2024-11-07T12:36:47,098 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into c4e03ff14a5e4a5493ab9f7e292055e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:47,098 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:47,098 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=13, startTime=1730983006650; duration=0sec 2024-11-07T12:36:47,098 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:47,098 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:47,098 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:47,100 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:47,100 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:47,100 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:47,100 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/caba2b6d0e95405d8bbd4f37cef34831, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f0d511650640424dae9df279dcbbab6e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/fafafac1b8af4388b5e9e900f7c00e0f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=36.0 K 2024-11-07T12:36:47,101 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting caba2b6d0e95405d8bbd4f37cef34831, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1730983004702 2024-11-07T12:36:47,101 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f0d511650640424dae9df279dcbbab6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983005184 2024-11-07T12:36:47,101 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting fafafac1b8af4388b5e9e900f7c00e0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730983005404 2024-11-07T12:36:47,104 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#165 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:47,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:36:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:47,105 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2da586f3ce744a448023fe98f102a46a is 175, key is test_row_0/A:col10/1730983005404/Put/seqid=0 2024-11-07T12:36:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:47,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:47,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:47,117 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:47,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742022_1198 (size=31617) 2024-11-07T12:36:47,118 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/83c062cb2f2347f082c6deee0c43aedd is 50, key is test_row_0/C:col10/1730983005404/Put/seqid=0 2024-11-07T12:36:47,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742023_1199 (size=12697) 2024-11-07T12:36:47,128 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/83c062cb2f2347f082c6deee0c43aedd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/83c062cb2f2347f082c6deee0c43aedd 2024-11-07T12:36:47,135 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into 83c062cb2f2347f082c6deee0c43aedd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:47,136 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:47,136 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=13, startTime=1730983006650; duration=0sec 2024-11-07T12:36:47,136 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:47,136 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:47,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:47,142 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075807873811c249e0a864a2dcafb183b2_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075807873811c249e0a864a2dcafb183b2_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:47,143 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/331932d3208046e4955e8f9b2db1dc84, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:47,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/331932d3208046e4955e8f9b2db1dc84 is 175, key is test_row_0/A:col10/1730983006691/Put/seqid=0 2024-11-07T12:36:47,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742024_1200 (size=39749) 2024-11-07T12:36:47,160 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/331932d3208046e4955e8f9b2db1dc84 2024-11-07T12:36:47,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/45f4ae9550e141319980212f15422d45 is 50, key is test_row_0/B:col10/1730983006691/Put/seqid=0 2024-11-07T12:36:47,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742025_1201 (size=12151) 2024-11-07T12:36:47,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/45f4ae9550e141319980212f15422d45 2024-11-07T12:36:47,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/f93d0b0486d24a78921cc2b1dd801b18 is 50, key is test_row_0/C:col10/1730983006691/Put/seqid=0 2024-11-07T12:36:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742026_1202 (size=12151) 2024-11-07T12:36:47,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/f93d0b0486d24a78921cc2b1dd801b18 2024-11-07T12:36:47,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/331932d3208046e4955e8f9b2db1dc84 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/331932d3208046e4955e8f9b2db1dc84 2024-11-07T12:36:47,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/331932d3208046e4955e8f9b2db1dc84, entries=200, sequenceid=255, filesize=38.8 K 2024-11-07T12:36:47,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/45f4ae9550e141319980212f15422d45 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/45f4ae9550e141319980212f15422d45 2024-11-07T12:36:47,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/45f4ae9550e141319980212f15422d45, entries=150, sequenceid=255, filesize=11.9 K 2024-11-07T12:36:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/f93d0b0486d24a78921cc2b1dd801b18 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f93d0b0486d24a78921cc2b1dd801b18 2024-11-07T12:36:47,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f93d0b0486d24a78921cc2b1dd801b18, entries=150, sequenceid=255, filesize=11.9 K 2024-11-07T12:36:47,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 292f80e0c9fe1beaa809aba371c91f47 in 559ms, sequenceid=255, compaction requested=false 2024-11-07T12:36:47,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:47,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:36:47,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:47,259 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:36:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:36:47,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110747217eb6f0c748ffb8c7c8870803dd63_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983006727/Put/seqid=0 2024-11-07T12:36:47,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742027_1203 (size=12454) 2024-11-07T12:36:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:47,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983067371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983067370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983067386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983067386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983067386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983067498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983067498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983067498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983067498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983067499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,525 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/2da586f3ce744a448023fe98f102a46a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2da586f3ce744a448023fe98f102a46a 2024-11-07T12:36:47,531 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into 2da586f3ce744a448023fe98f102a46a(size=30.9 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:47,531 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:47,532 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=13, startTime=1730983006649; duration=0sec 2024-11-07T12:36:47,532 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:47,532 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:36:47,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983067707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983067707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983067707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983067708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983067708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:47,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:47,722 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110747217eb6f0c748ffb8c7c8870803dd63_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110747217eb6f0c748ffb8c7c8870803dd63_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:47,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/d3ce94d7400c427299e031416255c707, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:47,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/d3ce94d7400c427299e031416255c707 is 175, key is test_row_0/A:col10/1730983006727/Put/seqid=0 2024-11-07T12:36:47,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742028_1204 (size=31255) 2024-11-07T12:36:48,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983068017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983068017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983068018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983068018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983068018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:36:48,149 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/d3ce94d7400c427299e031416255c707 2024-11-07T12:36:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/c56b771a995b4d4ca60d9fae26e8bd42 is 50, key is test_row_0/B:col10/1730983006727/Put/seqid=0 2024-11-07T12:36:48,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742029_1205 (size=12301) 2024-11-07T12:36:48,173 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/c56b771a995b4d4ca60d9fae26e8bd42 2024-11-07T12:36:48,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/11bd3d4b1bc847309fddc44531e3385e is 50, key is test_row_0/C:col10/1730983006727/Put/seqid=0 2024-11-07T12:36:48,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742030_1206 (size=12301) 2024-11-07T12:36:48,188 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/11bd3d4b1bc847309fddc44531e3385e 2024-11-07T12:36:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/d3ce94d7400c427299e031416255c707 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d3ce94d7400c427299e031416255c707 2024-11-07T12:36:48,200 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d3ce94d7400c427299e031416255c707, entries=150, sequenceid=281, filesize=30.5 K 2024-11-07T12:36:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/c56b771a995b4d4ca60d9fae26e8bd42 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c56b771a995b4d4ca60d9fae26e8bd42 2024-11-07T12:36:48,208 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c56b771a995b4d4ca60d9fae26e8bd42, entries=150, sequenceid=281, filesize=12.0 K 2024-11-07T12:36:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/11bd3d4b1bc847309fddc44531e3385e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/11bd3d4b1bc847309fddc44531e3385e 2024-11-07T12:36:48,217 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/11bd3d4b1bc847309fddc44531e3385e, entries=150, sequenceid=281, filesize=12.0 K 2024-11-07T12:36:48,219 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 292f80e0c9fe1beaa809aba371c91f47 in 960ms, sequenceid=281, compaction requested=true 2024-11-07T12:36:48,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:48,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:48,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-07T12:36:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-07T12:36:48,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-07T12:36:48,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2690 sec 2024-11-07T12:36:48,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2750 sec 2024-11-07T12:36:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:48,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:36:48,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b3f021e1a4e242a197261829a041d6bd_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983008531/Put/seqid=0 2024-11-07T12:36:48,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742031_1207 (size=12454) 2024-11-07T12:36:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983068563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983068565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983068567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983068571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983068573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983068678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983068678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983068678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983068687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983068687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983068888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983068888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983068888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983068897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983068897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:48,966 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:48,971 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b3f021e1a4e242a197261829a041d6bd_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b3f021e1a4e242a197261829a041d6bd_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:48,972 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/9521ed7dbb754de694334530a10596ed, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:48,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/9521ed7dbb754de694334530a10596ed is 175, key is test_row_0/A:col10/1730983008531/Put/seqid=0 2024-11-07T12:36:48,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742032_1208 (size=31255) 2024-11-07T12:36:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:36:49,101 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:49,101 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-07T12:36:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:36:49,104 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:49,104 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:49,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:49,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983069197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:36:49,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983069208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983069208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983069208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983069208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:36:49,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:49,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,377 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/9521ed7dbb754de694334530a10596ed 2024-11-07T12:36:49,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/b325b5f1792e4314b0e70322a355a8d2 is 50, key is test_row_0/B:col10/1730983008531/Put/seqid=0 2024-11-07T12:36:49,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742033_1209 (size=12301) 2024-11-07T12:36:49,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/b325b5f1792e4314b0e70322a355a8d2 2024-11-07T12:36:49,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/2ffc4f53cbab4b469adcf61191bfe71e is 50, key is test_row_0/C:col10/1730983008531/Put/seqid=0 2024-11-07T12:36:49,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742034_1210 (size=12301) 2024-11-07T12:36:49,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:36:49,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:49,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:36:49,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:36:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983069707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:36:49,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983069717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983069717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983069718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983069718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:36:49,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/2ffc4f53cbab4b469adcf61191bfe71e 2024-11-07T12:36:49,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/9521ed7dbb754de694334530a10596ed as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9521ed7dbb754de694334530a10596ed 2024-11-07T12:36:49,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9521ed7dbb754de694334530a10596ed, entries=150, sequenceid=297, filesize=30.5 K 2024-11-07T12:36:49,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/b325b5f1792e4314b0e70322a355a8d2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b325b5f1792e4314b0e70322a355a8d2 2024-11-07T12:36:49,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b325b5f1792e4314b0e70322a355a8d2, entries=150, sequenceid=297, filesize=12.0 K 2024-11-07T12:36:49,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/2ffc4f53cbab4b469adcf61191bfe71e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/2ffc4f53cbab4b469adcf61191bfe71e 2024-11-07T12:36:49,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/2ffc4f53cbab4b469adcf61191bfe71e, entries=150, sequenceid=297, filesize=12.0 K 2024-11-07T12:36:49,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 292f80e0c9fe1beaa809aba371c91f47 in 1312ms, sequenceid=297, compaction requested=true 2024-11-07T12:36:49,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:49,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:49,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:49,846 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:49,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:49,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:49,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:49,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:49,847 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:49,848 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:49,849 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:49,849 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,849 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c4e03ff14a5e4a5493ab9f7e292055e7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/45f4ae9550e141319980212f15422d45, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c56b771a995b4d4ca60d9fae26e8bd42, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b325b5f1792e4314b0e70322a355a8d2] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=48.3 K 2024-11-07T12:36:49,849 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133876 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:49,849 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:49,849 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,849 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2da586f3ce744a448023fe98f102a46a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/331932d3208046e4955e8f9b2db1dc84, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d3ce94d7400c427299e031416255c707, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9521ed7dbb754de694334530a10596ed] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=130.7 K 2024-11-07T12:36:49,849 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,849 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2da586f3ce744a448023fe98f102a46a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/331932d3208046e4955e8f9b2db1dc84, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d3ce94d7400c427299e031416255c707, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9521ed7dbb754de694334530a10596ed] 2024-11-07T12:36:49,850 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c4e03ff14a5e4a5493ab9f7e292055e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730983005404 2024-11-07T12:36:49,850 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da586f3ce744a448023fe98f102a46a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730983005404 2024-11-07T12:36:49,851 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f4ae9550e141319980212f15422d45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1730983006045 2024-11-07T12:36:49,851 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 331932d3208046e4955e8f9b2db1dc84, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1730983006045 2024-11-07T12:36:49,851 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c56b771a995b4d4ca60d9fae26e8bd42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1730983006722 2024-11-07T12:36:49,852 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b325b5f1792e4314b0e70322a355a8d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1730983007375 2024-11-07T12:36:49,852 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ce94d7400c427299e031416255c707, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1730983006722 2024-11-07T12:36:49,852 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9521ed7dbb754de694334530a10596ed, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1730983007375 2024-11-07T12:36:49,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:36:49,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,874 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T12:36:49,879 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:49,880 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/f2d3e94d65774253860fd221645e52db is 50, key is test_row_0/B:col10/1730983008531/Put/seqid=0 2024-11-07T12:36:49,886 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:49,905 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107eb2a206b7b2f4af9a88d1d1dc7dd0a35_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:49,907 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107eb2a206b7b2f4af9a88d1d1dc7dd0a35_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:49,907 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107eb2a206b7b2f4af9a88d1d1dc7dd0a35_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:49,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110754e589d38b6a4862a53da86066816cb3_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983008565/Put/seqid=0 2024-11-07T12:36:49,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742035_1211 (size=12983) 2024-11-07T12:36:49,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742036_1212 (size=4469) 2024-11-07T12:36:49,928 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#177 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:49,928 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/30776a4d3e6a42c0ac24ed873a067ba1 is 175, key is test_row_0/A:col10/1730983008531/Put/seqid=0 2024-11-07T12:36:49,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742037_1213 (size=12454) 2024-11-07T12:36:49,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742038_1214 (size=31903) 2024-11-07T12:36:49,963 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/30776a4d3e6a42c0ac24ed873a067ba1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/30776a4d3e6a42c0ac24ed873a067ba1 2024-11-07T12:36:49,970 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into 30776a4d3e6a42c0ac24ed873a067ba1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:49,970 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:49,970 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=12, startTime=1730983009846; duration=0sec 2024-11-07T12:36:49,970 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:49,970 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:49,970 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:36:49,971 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:36:49,971 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:49,972 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:49,972 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/83c062cb2f2347f082c6deee0c43aedd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f93d0b0486d24a78921cc2b1dd801b18, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/11bd3d4b1bc847309fddc44531e3385e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/2ffc4f53cbab4b469adcf61191bfe71e] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=48.3 K 2024-11-07T12:36:49,972 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83c062cb2f2347f082c6deee0c43aedd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730983005404 2024-11-07T12:36:49,973 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting f93d0b0486d24a78921cc2b1dd801b18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1730983006045 2024-11-07T12:36:49,974 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11bd3d4b1bc847309fddc44531e3385e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1730983006722 2024-11-07T12:36:49,974 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ffc4f53cbab4b469adcf61191bfe71e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1730983007375 2024-11-07T12:36:49,984 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:49,984 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a1cab7e52d01473e9013b4358d107f36 is 50, key is test_row_0/C:col10/1730983008531/Put/seqid=0 2024-11-07T12:36:49,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742039_1215 (size=12983) 2024-11-07T12:36:49,993 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a1cab7e52d01473e9013b4358d107f36 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a1cab7e52d01473e9013b4358d107f36 2024-11-07T12:36:50,000 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into a1cab7e52d01473e9013b4358d107f36(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:50,000 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:50,000 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=12, startTime=1730983009847; duration=0sec 2024-11-07T12:36:50,000 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:50,000 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:36:50,319 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/f2d3e94d65774253860fd221645e52db as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f2d3e94d65774253860fd221645e52db 2024-11-07T12:36:50,324 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into f2d3e94d65774253860fd221645e52db(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:50,324 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:50,324 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=12, startTime=1730983009846; duration=0sec 2024-11-07T12:36:50,325 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:50,325 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:50,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:50,339 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110754e589d38b6a4862a53da86066816cb3_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754e589d38b6a4862a53da86066816cb3_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:50,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/0a47065fe064408f965d3392ddcd3eee, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/0a47065fe064408f965d3392ddcd3eee is 175, key is test_row_0/A:col10/1730983008565/Put/seqid=0 2024-11-07T12:36:50,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742040_1216 (size=31255) 2024-11-07T12:36:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:50,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:50,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983070738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,746 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/0a47065fe064408f965d3392ddcd3eee 2024-11-07T12:36:50,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983070739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983070739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983070741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983070742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/3ca192b09d2c47b2b40e987aab60a50a is 50, key is test_row_0/B:col10/1730983008565/Put/seqid=0 2024-11-07T12:36:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742041_1217 (size=12301) 2024-11-07T12:36:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983070848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983070857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983070857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983070857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:50,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983070867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983071057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983071067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983071067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983071067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983071078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,163 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/3ca192b09d2c47b2b40e987aab60a50a 2024-11-07T12:36:51,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/102e2e2939bd431b84a18917410e2fe7 is 50, key is test_row_0/C:col10/1730983008565/Put/seqid=0 2024-11-07T12:36:51,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742042_1218 (size=12301) 2024-11-07T12:36:51,183 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/102e2e2939bd431b84a18917410e2fe7 2024-11-07T12:36:51,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/0a47065fe064408f965d3392ddcd3eee as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/0a47065fe064408f965d3392ddcd3eee 2024-11-07T12:36:51,198 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/0a47065fe064408f965d3392ddcd3eee, entries=150, sequenceid=318, filesize=30.5 K 2024-11-07T12:36:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/3ca192b09d2c47b2b40e987aab60a50a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/3ca192b09d2c47b2b40e987aab60a50a 2024-11-07T12:36:51,206 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/3ca192b09d2c47b2b40e987aab60a50a, entries=150, sequenceid=318, filesize=12.0 K 2024-11-07T12:36:51,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/102e2e2939bd431b84a18917410e2fe7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/102e2e2939bd431b84a18917410e2fe7 2024-11-07T12:36:51,217 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/102e2e2939bd431b84a18917410e2fe7, entries=150, sequenceid=318, filesize=12.0 K 2024-11-07T12:36:51,218 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 292f80e0c9fe1beaa809aba371c91f47 in 1345ms, sequenceid=318, compaction requested=false 2024-11-07T12:36:51,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-07T12:36:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-07T12:36:51,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T12:36:51,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1150 sec 2024-11-07T12:36:51,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1220 sec 2024-11-07T12:36:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:36:51,248 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:51,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-07T12:36:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:36:51,252 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:51,253 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:51,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:36:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:51,373 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:51,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T12:36:51,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075347ae6daf9a4ccf983b9850d3b769b9_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983010741/Put/seqid=0 2024-11-07T12:36:51,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742043_1219 (size=14994) 2024-11-07T12:36:51,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:36:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983071402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983071403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983071403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983071404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983071405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983071517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983071518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983071518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983071518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983071521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:36:51,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:51,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:36:51,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:36:51,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:51,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983071728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983071728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983071738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983071738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983071738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:51,797 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:51,802 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075347ae6daf9a4ccf983b9850d3b769b9_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075347ae6daf9a4ccf983b9850d3b769b9_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:51,803 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/b5b30d9daf02405da1090226c33d113e, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:51,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/b5b30d9daf02405da1090226c33d113e is 175, key is test_row_0/A:col10/1730983010741/Put/seqid=0 2024-11-07T12:36:51,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742044_1220 (size=39949) 2024-11-07T12:36:51,810 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/b5b30d9daf02405da1090226c33d113e 2024-11-07T12:36:51,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/89f45debf1884b239e783c7a402875bb is 50, key is test_row_0/B:col10/1730983010741/Put/seqid=0 2024-11-07T12:36:51,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742045_1221 (size=12301) 2024-11-07T12:36:51,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:36:51,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:51,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:51,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:36:52,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:36:52,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:52,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:52,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:52,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983072038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983072038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983072047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983072047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983072047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:36:52,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:52,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:52,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:52,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/89f45debf1884b239e783c7a402875bb 2024-11-07T12:36:52,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/aec0672ce8ce484bbcd1f84d77a7615e is 50, key is test_row_0/C:col10/1730983010741/Put/seqid=0 2024-11-07T12:36:52,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742046_1222 (size=12301) 2024-11-07T12:36:52,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/aec0672ce8ce484bbcd1f84d77a7615e 2024-11-07T12:36:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/b5b30d9daf02405da1090226c33d113e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b5b30d9daf02405da1090226c33d113e 2024-11-07T12:36:52,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b5b30d9daf02405da1090226c33d113e, entries=200, sequenceid=337, filesize=39.0 K 2024-11-07T12:36:52,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/89f45debf1884b239e783c7a402875bb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/89f45debf1884b239e783c7a402875bb 2024-11-07T12:36:52,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/89f45debf1884b239e783c7a402875bb, entries=150, sequenceid=337, filesize=12.0 K 2024-11-07T12:36:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/aec0672ce8ce484bbcd1f84d77a7615e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/aec0672ce8ce484bbcd1f84d77a7615e 2024-11-07T12:36:52,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/aec0672ce8ce484bbcd1f84d77a7615e, entries=150, sequenceid=337, filesize=12.0 K 2024-11-07T12:36:52,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 292f80e0c9fe1beaa809aba371c91f47 in 918ms, sequenceid=337, compaction requested=true 2024-11-07T12:36:52,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:52,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:52,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:52,292 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:52,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:52,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:52,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:52,292 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:52,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:52,293 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:52,293 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:52,293 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:52,293 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:52,293 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,293 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,293 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f2d3e94d65774253860fd221645e52db, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/3ca192b09d2c47b2b40e987aab60a50a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/89f45debf1884b239e783c7a402875bb] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=36.7 K 2024-11-07T12:36:52,293 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/30776a4d3e6a42c0ac24ed873a067ba1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/0a47065fe064408f965d3392ddcd3eee, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b5b30d9daf02405da1090226c33d113e] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=100.7 K 2024-11-07T12:36:52,293 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,293 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/30776a4d3e6a42c0ac24ed873a067ba1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/0a47065fe064408f965d3392ddcd3eee, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b5b30d9daf02405da1090226c33d113e] 2024-11-07T12:36:52,294 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d3e94d65774253860fd221645e52db, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1730983007375 2024-11-07T12:36:52,294 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30776a4d3e6a42c0ac24ed873a067ba1, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1730983007375 2024-11-07T12:36:52,294 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca192b09d2c47b2b40e987aab60a50a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1730983008560 2024-11-07T12:36:52,294 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a47065fe064408f965d3392ddcd3eee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1730983008560 2024-11-07T12:36:52,295 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f45debf1884b239e783c7a402875bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1730983010737 2024-11-07T12:36:52,295 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5b30d9daf02405da1090226c33d113e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1730983010737 2024-11-07T12:36:52,303 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:52,305 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:52,305 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/bc33aa42b4d448999c0d38dfbdff9afe is 50, key is test_row_0/B:col10/1730983010741/Put/seqid=0 2024-11-07T12:36:52,313 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107f3d47c0b300348c3b2f92955aaa57258_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:52,314 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107f3d47c0b300348c3b2f92955aaa57258_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:52,314 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f3d47c0b300348c3b2f92955aaa57258_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:52,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:36:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,325 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T12:36:52,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742047_1223 (size=13085) 2024-11-07T12:36:52,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742048_1224 (size=4469) 2024-11-07T12:36:52,340 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/bc33aa42b4d448999c0d38dfbdff9afe as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc33aa42b4d448999c0d38dfbdff9afe 2024-11-07T12:36:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107457e2054103143f19072b9c0c34662d3_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983011401/Put/seqid=0 2024-11-07T12:36:52,354 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into bc33aa42b4d448999c0d38dfbdff9afe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:52,354 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:52,354 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=13, startTime=1730983012292; duration=0sec 2024-11-07T12:36:52,354 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:52,354 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:52,354 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:52,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742049_1225 (size=12454) 2024-11-07T12:36:52,357 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:52,357 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:52,357 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,357 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a1cab7e52d01473e9013b4358d107f36, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/102e2e2939bd431b84a18917410e2fe7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/aec0672ce8ce484bbcd1f84d77a7615e] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=36.7 K 2024-11-07T12:36:52,358 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a1cab7e52d01473e9013b4358d107f36, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1730983007375 2024-11-07T12:36:52,358 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 102e2e2939bd431b84a18917410e2fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1730983008560 2024-11-07T12:36:52,359 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting aec0672ce8ce484bbcd1f84d77a7615e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1730983010737 2024-11-07T12:36:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:52,369 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107457e2054103143f19072b9c0c34662d3_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107457e2054103143f19072b9c0c34662d3_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:52,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/ec83a0271e93425dae27520e605d7eae, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/ec83a0271e93425dae27520e605d7eae is 175, key is test_row_0/A:col10/1730983011401/Put/seqid=0 2024-11-07T12:36:52,376 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:52,377 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/81f959de7a244be3aa8ead35cd72cab2 is 50, key is test_row_0/C:col10/1730983010741/Put/seqid=0 2024-11-07T12:36:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:36:52,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742051_1227 (size=13085) 2024-11-07T12:36:52,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742050_1226 (size=31255) 2024-11-07T12:36:52,399 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/ec83a0271e93425dae27520e605d7eae 2024-11-07T12:36:52,404 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/81f959de7a244be3aa8ead35cd72cab2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/81f959de7a244be3aa8ead35cd72cab2 2024-11-07T12:36:52,412 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into 81f959de7a244be3aa8ead35cd72cab2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:52,412 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:52,412 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=13, startTime=1730983012292; duration=0sec 2024-11-07T12:36:52,412 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:52,412 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:52,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/a176ae0a8ca74fcf9ae28169009e4782 is 50, key is test_row_0/B:col10/1730983011401/Put/seqid=0 2024-11-07T12:36:52,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742052_1228 (size=12301) 2024-11-07T12:36:52,426 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/a176ae0a8ca74fcf9ae28169009e4782 2024-11-07T12:36:52,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/af02ae42eddb4f0b81e90e1ce5c624a6 is 50, key is test_row_0/C:col10/1730983011401/Put/seqid=0 2024-11-07T12:36:52,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742053_1229 (size=12301) 2024-11-07T12:36:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:52,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:52,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983072563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983072565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983072565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983072566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983072567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983072678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983072678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983072679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983072679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983072679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,736 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#186 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:52,737 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/9f9b7b9d261b4cd9bf52528994c2279a is 175, key is test_row_0/A:col10/1730983010741/Put/seqid=0 2024-11-07T12:36:52,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742054_1230 (size=32005) 2024-11-07T12:36:52,856 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/af02ae42eddb4f0b81e90e1ce5c624a6 2024-11-07T12:36:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/ec83a0271e93425dae27520e605d7eae as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/ec83a0271e93425dae27520e605d7eae 2024-11-07T12:36:52,867 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/ec83a0271e93425dae27520e605d7eae, entries=150, sequenceid=357, filesize=30.5 K 2024-11-07T12:36:52,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/a176ae0a8ca74fcf9ae28169009e4782 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a176ae0a8ca74fcf9ae28169009e4782 2024-11-07T12:36:52,872 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a176ae0a8ca74fcf9ae28169009e4782, entries=150, sequenceid=357, filesize=12.0 K 2024-11-07T12:36:52,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/af02ae42eddb4f0b81e90e1ce5c624a6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/af02ae42eddb4f0b81e90e1ce5c624a6 2024-11-07T12:36:52,884 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/af02ae42eddb4f0b81e90e1ce5c624a6, entries=150, sequenceid=357, filesize=12.0 K 2024-11-07T12:36:52,885 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 292f80e0c9fe1beaa809aba371c91f47 in 559ms, sequenceid=357, compaction requested=false 2024-11-07T12:36:52,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:52,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:52,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-07T12:36:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-07T12:36:52,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T12:36:52,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6330 sec 2024-11-07T12:36:52,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6390 sec 2024-11-07T12:36:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:52,900 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:52,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T12:36:52,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079a4f0d1ca46f4a609fbeabf7103a573e_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983012564/Put/seqid=0 2024-11-07T12:36:52,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983072923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983072925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983072925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983072927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983072933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:52,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742055_1231 (size=14994) 2024-11-07T12:36:52,941 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:52,947 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079a4f0d1ca46f4a609fbeabf7103a573e_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a4f0d1ca46f4a609fbeabf7103a573e_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:52,948 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/6bb45eb9d4364844a3164892302b2841, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:52,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/6bb45eb9d4364844a3164892302b2841 is 175, key is test_row_0/A:col10/1730983012564/Put/seqid=0 2024-11-07T12:36:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742056_1232 (size=39949) 2024-11-07T12:36:53,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983073038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983073038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983073038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983073038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983073038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,148 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/9f9b7b9d261b4cd9bf52528994c2279a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9f9b7b9d261b4cd9bf52528994c2279a 2024-11-07T12:36:53,153 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into 9f9b7b9d261b4cd9bf52528994c2279a(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:53,153 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:53,153 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=13, startTime=1730983012291; duration=0sec 2024-11-07T12:36:53,153 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:53,154 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:53,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983073248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983073248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983073248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983073248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983073248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,359 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/6bb45eb9d4364844a3164892302b2841 2024-11-07T12:36:53,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/c8fcaab780de44368febc9809807861f is 50, key is test_row_0/B:col10/1730983012564/Put/seqid=0 2024-11-07T12:36:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742057_1233 (size=12301) 2024-11-07T12:36:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:36:53,397 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:53,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-07T12:36:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:36:53,400 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:53,401 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:53,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:36:53,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T12:36:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:53,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:53,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:53,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:53,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983073557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983073557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983073558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983073558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983073558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:53,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T12:36:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:53,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:36:53,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/c8fcaab780de44368febc9809807861f 2024-11-07T12:36:53,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/ab6928f142964c3a9076e927e03bfa83 is 50, key is test_row_0/C:col10/1730983012564/Put/seqid=0 2024-11-07T12:36:53,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742058_1234 (size=12301) 2024-11-07T12:36:53,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T12:36:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:53,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:54,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T12:36:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:36:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983074067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:54,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983074077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:54,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983074077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:54,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983074077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:54,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983074077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:54,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:54,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T12:36:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:54,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/ab6928f142964c3a9076e927e03bfa83 2024-11-07T12:36:54,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/6bb45eb9d4364844a3164892302b2841 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/6bb45eb9d4364844a3164892302b2841 2024-11-07T12:36:54,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/6bb45eb9d4364844a3164892302b2841, entries=200, sequenceid=376, filesize=39.0 K 2024-11-07T12:36:54,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/c8fcaab780de44368febc9809807861f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c8fcaab780de44368febc9809807861f 2024-11-07T12:36:54,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c8fcaab780de44368febc9809807861f, entries=150, sequenceid=376, filesize=12.0 K 2024-11-07T12:36:54,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/ab6928f142964c3a9076e927e03bfa83 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/ab6928f142964c3a9076e927e03bfa83 2024-11-07T12:36:54,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/ab6928f142964c3a9076e927e03bfa83, entries=150, sequenceid=376, filesize=12.0 K 2024-11-07T12:36:54,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 292f80e0c9fe1beaa809aba371c91f47 in 1316ms, sequenceid=376, compaction requested=true 2024-11-07T12:36:54,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:54,217 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:54,217 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:54,218 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:54,218 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:54,218 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:54,218 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:54,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:54,218 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:54,218 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:54,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:54,218 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9f9b7b9d261b4cd9bf52528994c2279a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/ec83a0271e93425dae27520e605d7eae, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/6bb45eb9d4364844a3164892302b2841] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=100.8 K 2024-11-07T12:36:54,218 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc33aa42b4d448999c0d38dfbdff9afe, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a176ae0a8ca74fcf9ae28169009e4782, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c8fcaab780de44368febc9809807861f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=36.8 K 2024-11-07T12:36:54,218 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,219 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9f9b7b9d261b4cd9bf52528994c2279a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/ec83a0271e93425dae27520e605d7eae, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/6bb45eb9d4364844a3164892302b2841] 2024-11-07T12:36:54,219 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bc33aa42b4d448999c0d38dfbdff9afe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1730983010737 2024-11-07T12:36:54,220 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f9b7b9d261b4cd9bf52528994c2279a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1730983010737 2024-11-07T12:36:54,220 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a176ae0a8ca74fcf9ae28169009e4782, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1730983011401 2024-11-07T12:36:54,220 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec83a0271e93425dae27520e605d7eae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1730983011401 2024-11-07T12:36:54,220 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c8fcaab780de44368febc9809807861f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983012564 2024-11-07T12:36:54,221 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bb45eb9d4364844a3164892302b2841, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983012563 2024-11-07T12:36:54,241 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:54,243 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:54,244 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/6e89a9dad03a4f67b6216f6094d32c75 is 50, key is test_row_0/B:col10/1730983012564/Put/seqid=0 2024-11-07T12:36:54,253 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107eb24767f59ef4cb1b0a525a61777b276_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:54,255 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107eb24767f59ef4cb1b0a525a61777b276_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:54,256 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107eb24767f59ef4cb1b0a525a61777b276_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:54,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742059_1235 (size=13187) 2024-11-07T12:36:54,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742060_1236 (size=4469) 2024-11-07T12:36:54,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T12:36:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,320 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T12:36:54,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f75ac0a465134db38e14bfba33ecb427_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983012924/Put/seqid=0 2024-11-07T12:36:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742061_1237 (size=12454) 2024-11-07T12:36:54,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,349 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f75ac0a465134db38e14bfba33ecb427_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f75ac0a465134db38e14bfba33ecb427_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:54,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/bbdf0d5b69f348d5bc031ab5895b2b4c, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:54,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/bbdf0d5b69f348d5bc031ab5895b2b4c is 175, key is test_row_0/A:col10/1730983012924/Put/seqid=0 2024-11-07T12:36:54,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742062_1238 (size=31255) 2024-11-07T12:36:54,355 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/bbdf0d5b69f348d5bc031ab5895b2b4c 2024-11-07T12:36:54,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/30ddf831e1a94c53b4c76447aec6dfce is 50, key is test_row_0/B:col10/1730983012924/Put/seqid=0 2024-11-07T12:36:54,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742063_1239 (size=12301) 2024-11-07T12:36:54,370 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/30ddf831e1a94c53b4c76447aec6dfce 2024-11-07T12:36:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/05ea53d58227466a8ca729ec82414337 is 50, key is test_row_0/C:col10/1730983012924/Put/seqid=0 2024-11-07T12:36:54,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742064_1240 (size=12301) 2024-11-07T12:36:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:36:54,693 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/6e89a9dad03a4f67b6216f6094d32c75 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/6e89a9dad03a4f67b6216f6094d32c75 2024-11-07T12:36:54,694 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#194 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:54,695 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/dcb87f3f818e405c8eaeb0a3065fa9b7 is 175, key is test_row_0/A:col10/1730983012564/Put/seqid=0 2024-11-07T12:36:54,699 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into 6e89a9dad03a4f67b6216f6094d32c75(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:54,699 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:54,699 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=13, startTime=1730983014217; duration=0sec 2024-11-07T12:36:54,699 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:54,699 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:54,699 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:54,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742065_1241 (size=32107) 2024-11-07T12:36:54,707 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:54,708 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:54,708 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,708 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/81f959de7a244be3aa8ead35cd72cab2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/af02ae42eddb4f0b81e90e1ce5c624a6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/ab6928f142964c3a9076e927e03bfa83] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=36.8 K 2024-11-07T12:36:54,708 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f959de7a244be3aa8ead35cd72cab2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1730983010737 2024-11-07T12:36:54,709 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting af02ae42eddb4f0b81e90e1ce5c624a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1730983011401 2024-11-07T12:36:54,710 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6928f142964c3a9076e927e03bfa83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983012564 2024-11-07T12:36:54,714 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/dcb87f3f818e405c8eaeb0a3065fa9b7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/dcb87f3f818e405c8eaeb0a3065fa9b7 2024-11-07T12:36:54,721 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:54,721 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into dcb87f3f818e405c8eaeb0a3065fa9b7(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:54,721 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:54,721 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=13, startTime=1730983014217; duration=0sec 2024-11-07T12:36:54,721 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/eae9d9f0253a4b43a00bd580b85db81c is 50, key is test_row_0/C:col10/1730983012564/Put/seqid=0 2024-11-07T12:36:54,721 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:54,721 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:54,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742066_1242 (size=13187) 2024-11-07T12:36:54,734 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/eae9d9f0253a4b43a00bd580b85db81c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/eae9d9f0253a4b43a00bd580b85db81c 2024-11-07T12:36:54,739 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into eae9d9f0253a4b43a00bd580b85db81c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:54,739 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:54,739 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=13, startTime=1730983014218; duration=0sec 2024-11-07T12:36:54,739 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:54,739 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:54,791 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/05ea53d58227466a8ca729ec82414337 2024-11-07T12:36:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/bbdf0d5b69f348d5bc031ab5895b2b4c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/bbdf0d5b69f348d5bc031ab5895b2b4c 2024-11-07T12:36:54,800 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/bbdf0d5b69f348d5bc031ab5895b2b4c, entries=150, sequenceid=396, filesize=30.5 K 2024-11-07T12:36:54,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/30ddf831e1a94c53b4c76447aec6dfce as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/30ddf831e1a94c53b4c76447aec6dfce 2024-11-07T12:36:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,805 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/30ddf831e1a94c53b4c76447aec6dfce, entries=150, sequenceid=396, filesize=12.0 K 2024-11-07T12:36:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/05ea53d58227466a8ca729ec82414337 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/05ea53d58227466a8ca729ec82414337 2024-11-07T12:36:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,810 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/05ea53d58227466a8ca729ec82414337, entries=150, sequenceid=396, filesize=12.0 K 2024-11-07T12:36:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,811 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 292f80e0c9fe1beaa809aba371c91f47 in 491ms, sequenceid=396, compaction requested=false 2024-11-07T12:36:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-07T12:36:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-07T12:36:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-07T12:36:54,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-07T12:36:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4180 sec 2024-11-07T12:36:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:55,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:36:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110704ee0f227f194fc2ada8a47ce2ec2d67_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983015126/Put/seqid=0 2024-11-07T12:36:55,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983075173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983075174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983075175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983075177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983075178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742068_1244 (size=25158) 2024-11-07T12:36:55,193 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:55,198 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110704ee0f227f194fc2ada8a47ce2ec2d67_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110704ee0f227f194fc2ada8a47ce2ec2d67_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:55,199 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/d9fbbafb4d41448abe1d069141388e0f, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:55,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/d9fbbafb4d41448abe1d069141388e0f is 175, key is test_row_0/A:col10/1730983015126/Put/seqid=0 2024-11-07T12:36:55,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742067_1243 (size=74795) 2024-11-07T12:36:55,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983075288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983075288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983075288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983075288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983075297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983075497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983075497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983075498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983075498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983075508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:36:55,547 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:55,548 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-07T12:36:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:36:55,550 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:55,551 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:55,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:55,602 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/d9fbbafb4d41448abe1d069141388e0f 2024-11-07T12:36:55,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/2a116937a3c94751a251440ab0488927 is 50, key is test_row_0/B:col10/1730983015126/Put/seqid=0 2024-11-07T12:36:55,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742069_1245 (size=12301) 2024-11-07T12:36:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:36:55,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:36:55,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:55,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:55,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983075807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983075808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983075808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983075808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983075817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:55,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:55,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:55,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:36:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:55,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:36:56,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:36:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/2a116937a3c94751a251440ab0488927 2024-11-07T12:36:56,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a867640cc2e94a26b588f9bdf9aa97cf is 50, key is test_row_0/C:col10/1730983015126/Put/seqid=0 2024-11-07T12:36:56,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742070_1246 (size=12301) 2024-11-07T12:36:56,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:36:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:36:56,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:36:56,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:56,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:56,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983076318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983076318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:56,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983076318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983076318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:56,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983076337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:56,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a867640cc2e94a26b588f9bdf9aa97cf 2024-11-07T12:36:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/d9fbbafb4d41448abe1d069141388e0f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d9fbbafb4d41448abe1d069141388e0f 2024-11-07T12:36:56,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d9fbbafb4d41448abe1d069141388e0f, entries=400, sequenceid=410, filesize=73.0 K 2024-11-07T12:36:56,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/2a116937a3c94751a251440ab0488927 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/2a116937a3c94751a251440ab0488927 2024-11-07T12:36:56,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/2a116937a3c94751a251440ab0488927, entries=150, sequenceid=410, filesize=12.0 K 2024-11-07T12:36:56,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a867640cc2e94a26b588f9bdf9aa97cf as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a867640cc2e94a26b588f9bdf9aa97cf 2024-11-07T12:36:56,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a867640cc2e94a26b588f9bdf9aa97cf, entries=150, sequenceid=410, filesize=12.0 K 2024-11-07T12:36:56,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 292f80e0c9fe1beaa809aba371c91f47 in 1324ms, sequenceid=410, compaction requested=true 2024-11-07T12:36:56,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:56,455 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:36:56,455 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:56,456 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138157 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:56,456 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:56,456 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,456 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/dcb87f3f818e405c8eaeb0a3065fa9b7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/bbdf0d5b69f348d5bc031ab5895b2b4c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d9fbbafb4d41448abe1d069141388e0f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=134.9 K 2024-11-07T12:36:56,456 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:56,456 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,456 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/dcb87f3f818e405c8eaeb0a3065fa9b7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/bbdf0d5b69f348d5bc031ab5895b2b4c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d9fbbafb4d41448abe1d069141388e0f] 2024-11-07T12:36:56,457 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:56,457 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,457 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/6e89a9dad03a4f67b6216f6094d32c75, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/30ddf831e1a94c53b4c76447aec6dfce, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/2a116937a3c94751a251440ab0488927] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=36.9 K 2024-11-07T12:36:56,457 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcb87f3f818e405c8eaeb0a3065fa9b7, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983012564 2024-11-07T12:36:56,457 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e89a9dad03a4f67b6216f6094d32c75, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983012564 2024-11-07T12:36:56,458 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ddf831e1a94c53b4c76447aec6dfce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730983012922 2024-11-07T12:36:56,458 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbdf0d5b69f348d5bc031ab5895b2b4c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730983012922 2024-11-07T12:36:56,458 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a116937a3c94751a251440ab0488927, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730983015115 2024-11-07T12:36:56,458 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9fbbafb4d41448abe1d069141388e0f, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730983015113 2024-11-07T12:36:56,471 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:56,491 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#203 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:56,492 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/14fdf7ebd173482a9eb54079b4dedb61 is 50, key is test_row_0/B:col10/1730983015126/Put/seqid=0 2024-11-07T12:36:56,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:36:56,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,495 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:36:56,507 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107188044fcd6f040ed8c926a049981b797_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:56,510 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107188044fcd6f040ed8c926a049981b797_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:56,511 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107188044fcd6f040ed8c926a049981b797_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:56,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f59870844a944a1482d10cd5243b08ad_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983015171/Put/seqid=0 2024-11-07T12:36:56,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742071_1247 (size=13289) 2024-11-07T12:36:56,529 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/14fdf7ebd173482a9eb54079b4dedb61 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/14fdf7ebd173482a9eb54079b4dedb61 2024-11-07T12:36:56,540 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into 14fdf7ebd173482a9eb54079b4dedb61(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:56,541 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:56,541 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=13, startTime=1730983016455; duration=0sec 2024-11-07T12:36:56,541 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:56,541 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:56,541 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:56,542 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:56,542 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:56,542 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:56,542 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/eae9d9f0253a4b43a00bd580b85db81c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/05ea53d58227466a8ca729ec82414337, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a867640cc2e94a26b588f9bdf9aa97cf] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=36.9 K 2024-11-07T12:36:56,543 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting eae9d9f0253a4b43a00bd580b85db81c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983012564 2024-11-07T12:36:56,544 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ea53d58227466a8ca729ec82414337, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1730983012922 2024-11-07T12:36:56,544 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a867640cc2e94a26b588f9bdf9aa97cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730983015115 2024-11-07T12:36:56,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742072_1248 (size=4469) 2024-11-07T12:36:56,567 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:56,567 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/0ac61bd424a544a49d905fb7476c2216 is 50, key is test_row_0/C:col10/1730983015126/Put/seqid=0 2024-11-07T12:36:56,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742073_1249 (size=12454) 2024-11-07T12:36:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:56,579 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f59870844a944a1482d10cd5243b08ad_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f59870844a944a1482d10cd5243b08ad_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/cf556e7e29d943f193df6b76b8613d3e, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/cf556e7e29d943f193df6b76b8613d3e is 175, key is test_row_0/A:col10/1730983015171/Put/seqid=0 2024-11-07T12:36:56,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742075_1251 (size=31255) 2024-11-07T12:36:56,590 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=436, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/cf556e7e29d943f193df6b76b8613d3e 2024-11-07T12:36:56,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742074_1250 (size=13289) 2024-11-07T12:36:56,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/f3877883f727456da61c345f5df50398 is 50, key is test_row_0/B:col10/1730983015171/Put/seqid=0 2024-11-07T12:36:56,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742076_1252 (size=12301) 2024-11-07T12:36:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:36:56,957 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#204 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:56,958 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/908d72ad724c4d68899f31c8b326e3ff is 175, key is test_row_0/A:col10/1730983015126/Put/seqid=0 2024-11-07T12:36:56,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742077_1253 (size=32209) 2024-11-07T12:36:57,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/0ac61bd424a544a49d905fb7476c2216 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0ac61bd424a544a49d905fb7476c2216 2024-11-07T12:36:57,014 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/f3877883f727456da61c345f5df50398 2024-11-07T12:36:57,016 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into 0ac61bd424a544a49d905fb7476c2216(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:57,016 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:57,016 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=13, startTime=1730983016455; duration=0sec 2024-11-07T12:36:57,016 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:57,017 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:36:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/0455e59e4bf7432e9f3be387e5fe3eac is 50, key is test_row_0/C:col10/1730983015171/Put/seqid=0 2024-11-07T12:36:57,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742078_1254 (size=12301) 2024-11-07T12:36:57,040 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/0455e59e4bf7432e9f3be387e5fe3eac 2024-11-07T12:36:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/cf556e7e29d943f193df6b76b8613d3e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/cf556e7e29d943f193df6b76b8613d3e 2024-11-07T12:36:57,052 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/cf556e7e29d943f193df6b76b8613d3e, entries=150, sequenceid=436, filesize=30.5 K 2024-11-07T12:36:57,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/f3877883f727456da61c345f5df50398 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f3877883f727456da61c345f5df50398 2024-11-07T12:36:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,065 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f3877883f727456da61c345f5df50398, entries=150, sequenceid=436, filesize=12.0 K 2024-11-07T12:36:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/0455e59e4bf7432e9f3be387e5fe3eac as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0455e59e4bf7432e9f3be387e5fe3eac 2024-11-07T12:36:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,079 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0455e59e4bf7432e9f3be387e5fe3eac, entries=150, sequenceid=436, filesize=12.0 K 2024-11-07T12:36:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,080 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 292f80e0c9fe1beaa809aba371c91f47 in 585ms, sequenceid=436, compaction requested=false 2024-11-07T12:36:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-07T12:36:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-07T12:36:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T12:36:57,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5310 sec 2024-11-07T12:36:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5370 sec 2024-11-07T12:36:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,373 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/908d72ad724c4d68899f31c8b326e3ff as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/908d72ad724c4d68899f31c8b326e3ff 2024-11-07T12:36:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,379 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into 908d72ad724c4d68899f31c8b326e3ff(size=31.5 K), total size for store is 62.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:57,379 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:57,379 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=13, startTime=1730983016455; duration=0sec 2024-11-07T12:36:57,379 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:57,379 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:57,387 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:36:57,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:36:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d5c435b5978f419682001df38a169347_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983017373/Put/seqid=0 2024-11-07T12:36:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742079_1255 (size=25158) 2024-11-07T12:36:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983077441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983077447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983077449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983077449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983077449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983077557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983077557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983077557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983077557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983077558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:36:57,698 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:36:57,698 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:36:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-07T12:36:57,701 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:36:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:36:57,702 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:36:57,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:36:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983077768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983077768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983077768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983077768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983077768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:36:57,829 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:57,835 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d5c435b5978f419682001df38a169347_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d5c435b5978f419682001df38a169347_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:57,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/86768a13eb3a491f91b44f4d8fa345d3, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:57,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/86768a13eb3a491f91b44f4d8fa345d3 is 175, key is test_row_0/A:col10/1730983017373/Put/seqid=0 2024-11-07T12:36:57,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742080_1256 (size=74795) 2024-11-07T12:36:57,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:57,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:57,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:57,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:57,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:57,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:57,857 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-07T12:36:57,857 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:57,857 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:57,857 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:57,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:57,858 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-07T12:36:57,858 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:57,858 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:57,858 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:57,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:57,858 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-07T12:36:57,859 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:57,859 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:57,859 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:57,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:57,859 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-07T12:36:57,859 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:57,859 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:57,859 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:57,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:58,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:36:58,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983078078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983078078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983078078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983078078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983078087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:58,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,247 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/86768a13eb3a491f91b44f4d8fa345d3 2024-11-07T12:36:58,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/55e01785e841424ea40728477f8e6e47 is 50, key is test_row_0/B:col10/1730983017373/Put/seqid=0 2024-11-07T12:36:58,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742081_1257 (size=12301) 2024-11-07T12:36:58,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:36:58,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56198 deadline: 1730983078588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56236 deadline: 1730983078588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56234 deadline: 1730983078588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56206 deadline: 1730983078588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:36:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56250 deadline: 1730983078597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 2024-11-07T12:36:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:36:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:36:58,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/55e01785e841424ea40728477f8e6e47 2024-11-07T12:36:58,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a6c91732bf5249f7b7647a2fd1185944 is 50, key is test_row_0/C:col10/1730983017373/Put/seqid=0 2024-11-07T12:36:58,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742082_1258 (size=12301) 2024-11-07T12:36:58,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:58,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:58,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:36:58,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:58,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:58,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:58,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:59,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a6c91732bf5249f7b7647a2fd1185944 2024-11-07T12:36:59,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/86768a13eb3a491f91b44f4d8fa345d3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/86768a13eb3a491f91b44f4d8fa345d3 2024-11-07T12:36:59,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/86768a13eb3a491f91b44f4d8fa345d3, entries=400, sequenceid=450, filesize=73.0 K 2024-11-07T12:36:59,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/55e01785e841424ea40728477f8e6e47 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/55e01785e841424ea40728477f8e6e47 2024-11-07T12:36:59,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:59,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:59,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/55e01785e841424ea40728477f8e6e47, entries=150, sequenceid=450, filesize=12.0 K 2024-11-07T12:36:59,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:59,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:59,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:59,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:59,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/a6c91732bf5249f7b7647a2fd1185944 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a6c91732bf5249f7b7647a2fd1185944 2024-11-07T12:36:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:36:59,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a6c91732bf5249f7b7647a2fd1185944, entries=150, sequenceid=450, filesize=12.0 K 2024-11-07T12:36:59,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 292f80e0c9fe1beaa809aba371c91f47 in 1699ms, sequenceid=450, compaction requested=true 2024-11-07T12:36:59,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:59,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:36:59,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:59,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:36:59,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:59,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 292f80e0c9fe1beaa809aba371c91f47:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:36:59,086 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:59,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:59,086 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:59,087 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:59,087 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/B is initiating minor compaction (all files) 2024-11-07T12:36:59,087 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:59,087 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/B in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:59,088 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/A is initiating minor compaction (all files) 2024-11-07T12:36:59,088 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/14fdf7ebd173482a9eb54079b4dedb61, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f3877883f727456da61c345f5df50398, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/55e01785e841424ea40728477f8e6e47] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=37.0 K 2024-11-07T12:36:59,088 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/A in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:59,088 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/908d72ad724c4d68899f31c8b326e3ff, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/cf556e7e29d943f193df6b76b8613d3e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/86768a13eb3a491f91b44f4d8fa345d3] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=135.0 K 2024-11-07T12:36:59,088 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:59,088 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/908d72ad724c4d68899f31c8b326e3ff, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/cf556e7e29d943f193df6b76b8613d3e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/86768a13eb3a491f91b44f4d8fa345d3] 2024-11-07T12:36:59,088 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 14fdf7ebd173482a9eb54079b4dedb61, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730983015115 2024-11-07T12:36:59,088 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f3877883f727456da61c345f5df50398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730983015165 2024-11-07T12:36:59,088 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908d72ad724c4d68899f31c8b326e3ff, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730983015115 2024-11-07T12:36:59,089 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 55e01785e841424ea40728477f8e6e47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983017371 2024-11-07T12:36:59,089 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf556e7e29d943f193df6b76b8613d3e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730983015165 2024-11-07T12:36:59,089 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86768a13eb3a491f91b44f4d8fa345d3, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983017359 2024-11-07T12:36:59,095 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:59,096 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#B#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:59,097 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107110557b99d4943278397052796424156_292f80e0c9fe1beaa809aba371c91f47 store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:59,097 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/359bc0a2017a4d0fb20e61d0129497a4 is 50, key is test_row_0/B:col10/1730983017373/Put/seqid=0 2024-11-07T12:36:59,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742083_1259 (size=13391) 2024-11-07T12:36:59,113 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107110557b99d4943278397052796424156_292f80e0c9fe1beaa809aba371c91f47, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:59,113 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107110557b99d4943278397052796424156_292f80e0c9fe1beaa809aba371c91f47 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:59,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742084_1260 (size=4469) 2024-11-07T12:36:59,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:36:59,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:59,234 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:36:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110701c3c998e4a74297b048a4afdc6ad2c1_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983017447/Put/seqid=0 2024-11-07T12:36:59,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742085_1261 (size=12454) 2024-11-07T12:36:59,506 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/359bc0a2017a4d0fb20e61d0129497a4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/359bc0a2017a4d0fb20e61d0129497a4 2024-11-07T12:36:59,511 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/B of 292f80e0c9fe1beaa809aba371c91f47 into 359bc0a2017a4d0fb20e61d0129497a4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:59,511 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:59,511 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/B, priority=13, startTime=1730983019086; duration=0sec 2024-11-07T12:36:59,511 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:36:59,511 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:B 2024-11-07T12:36:59,511 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:36:59,512 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:36:59,512 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 292f80e0c9fe1beaa809aba371c91f47/C is initiating minor compaction (all files) 2024-11-07T12:36:59,512 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 292f80e0c9fe1beaa809aba371c91f47/C in TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:36:59,512 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0ac61bd424a544a49d905fb7476c2216, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0455e59e4bf7432e9f3be387e5fe3eac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a6c91732bf5249f7b7647a2fd1185944] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp, totalSize=37.0 K 2024-11-07T12:36:59,512 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ac61bd424a544a49d905fb7476c2216, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730983015115 2024-11-07T12:36:59,512 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 0455e59e4bf7432e9f3be387e5fe3eac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1730983015165 2024-11-07T12:36:59,513 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c91732bf5249f7b7647a2fd1185944, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983017371 2024-11-07T12:36:59,518 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#A#compaction#213 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:59,519 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/872ad2cf79704e48b795663e702f68d5 is 175, key is test_row_0/A:col10/1730983017373/Put/seqid=0 2024-11-07T12:36:59,519 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 292f80e0c9fe1beaa809aba371c91f47#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:36:59,520 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/0db50381015842ab8518f9d2da84cfea is 50, key is test_row_0/C:col10/1730983017373/Put/seqid=0 2024-11-07T12:36:59,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742086_1262 (size=32311) 2024-11-07T12:36:59,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742087_1263 (size=13391) 2024-11-07T12:36:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:59,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. as already flushing 2024-11-07T12:36:59,599 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-07T12:36:59,599 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-07T12:36:59,599 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:59,599 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:59,599 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,600 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,600 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,600 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:59,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:59,609 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-11-07T12:36:59,609 INFO [Thread-667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-667. 2024-11-07T12:36:59,609 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-07T12:36:59,609 DEBUG [Thread-667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:59,609 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:59,609 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:36:59,609 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,609 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,609 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,609 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,609 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,609 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:36:59,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:59,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:59,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:36:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:36:59,651 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110701c3c998e4a74297b048a4afdc6ad2c1_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110701c3c998e4a74297b048a4afdc6ad2c1_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:36:59,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/7bdee6cba73045279c9f9f7ffa9bf32c, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:36:59,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/7bdee6cba73045279c9f9f7ffa9bf32c is 175, key is test_row_0/A:col10/1730983017447/Put/seqid=0 2024-11-07T12:36:59,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742088_1264 (size=31255) 2024-11-07T12:36:59,656 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=475, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/7bdee6cba73045279c9f9f7ffa9bf32c 2024-11-07T12:36:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/268cdf0e8ef442578a05016d0b8bcc4c is 50, key is test_row_0/B:col10/1730983017447/Put/seqid=0 2024-11-07T12:36:59,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742089_1265 (size=12301) 2024-11-07T12:36:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:36:59,928 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/0db50381015842ab8518f9d2da84cfea as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0db50381015842ab8518f9d2da84cfea 2024-11-07T12:36:59,928 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/872ad2cf79704e48b795663e702f68d5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/872ad2cf79704e48b795663e702f68d5 2024-11-07T12:36:59,934 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/C of 292f80e0c9fe1beaa809aba371c91f47 into 0db50381015842ab8518f9d2da84cfea(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:59,934 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 292f80e0c9fe1beaa809aba371c91f47/A of 292f80e0c9fe1beaa809aba371c91f47 into 872ad2cf79704e48b795663e702f68d5(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:36:59,934 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:59,934 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:36:59,934 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/C, priority=13, startTime=1730983019086; duration=0sec 2024-11-07T12:36:59,934 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., storeName=292f80e0c9fe1beaa809aba371c91f47/A, priority=13, startTime=1730983019086; duration=0sec 2024-11-07T12:36:59,935 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:59,935 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:A 2024-11-07T12:36:59,935 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:36:59,935 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 292f80e0c9fe1beaa809aba371c91f47:C 2024-11-07T12:37:00,066 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/268cdf0e8ef442578a05016d0b8bcc4c 2024-11-07T12:37:00,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/19da44c7899e47c7ac3d12559dcb1018 is 50, key is test_row_0/C:col10/1730983017447/Put/seqid=0 2024-11-07T12:37:00,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742090_1266 (size=12301) 2024-11-07T12:37:00,246 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:37:00,480 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/19da44c7899e47c7ac3d12559dcb1018 2024-11-07T12:37:00,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/7bdee6cba73045279c9f9f7ffa9bf32c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/7bdee6cba73045279c9f9f7ffa9bf32c 2024-11-07T12:37:00,488 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/7bdee6cba73045279c9f9f7ffa9bf32c, entries=150, sequenceid=475, filesize=30.5 K 2024-11-07T12:37:00,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/268cdf0e8ef442578a05016d0b8bcc4c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/268cdf0e8ef442578a05016d0b8bcc4c 2024-11-07T12:37:00,493 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/268cdf0e8ef442578a05016d0b8bcc4c, entries=150, sequenceid=475, filesize=12.0 K 2024-11-07T12:37:00,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/19da44c7899e47c7ac3d12559dcb1018 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19da44c7899e47c7ac3d12559dcb1018 2024-11-07T12:37:00,497 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19da44c7899e47c7ac3d12559dcb1018, entries=150, sequenceid=475, filesize=12.0 K 2024-11-07T12:37:00,498 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 292f80e0c9fe1beaa809aba371c91f47 in 1264ms, sequenceid=475, compaction requested=false 2024-11-07T12:37:00,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 292f80e0c9fe1beaa809aba371c91f47: 2024-11-07T12:37:00,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:37:00,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-07T12:37:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-07T12:37:00,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T12:37:00,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7970 sec 2024-11-07T12:37:00,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.8030 sec 2024-11-07T12:37:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:37:01,858 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3021 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3015 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1356 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4067 rows 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1361 2024-11-07T12:37:01,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4081 rows 2024-11-07T12:37:01,859 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:37:01,859 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:37:01,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:01,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:01,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:01,860 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T12:37:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:37:01,863 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983021863"}]},"ts":"1730983021863"} 2024-11-07T12:37:01,864 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:37:01,864 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:37:01,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:37:01,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, UNASSIGN}] 2024-11-07T12:37:01,867 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, UNASSIGN 2024-11-07T12:37:01,867 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=292f80e0c9fe1beaa809aba371c91f47, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:01,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, UNASSIGN because future has completed 2024-11-07T12:37:01,869 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:37:01,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:37:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:37:02,021 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:02,021 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:37:02,021 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 292f80e0c9fe1beaa809aba371c91f47, disabling compactions & flushes 2024-11-07T12:37:02,021 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:37:02,021 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:37:02,021 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. after waiting 0 ms 2024-11-07T12:37:02,021 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:37:02,021 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 292f80e0c9fe1beaa809aba371c91f47 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T12:37:02,028 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aaa8be2155e943cfbe877b37aecb709e_292f80e0c9fe1beaa809aba371c91f47 is 50, key is test_row_0/A:col10/1730983019598/Put/seqid=0 2024-11-07T12:37:02,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742091_1267 (size=12454) 2024-11-07T12:37:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:37:02,432 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:02,437 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aaa8be2155e943cfbe877b37aecb709e_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aaa8be2155e943cfbe877b37aecb709e_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:02,438 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/f87eebcfb7ed49029e962842a39b8201, store: [table=TestAcidGuarantees family=A region=292f80e0c9fe1beaa809aba371c91f47] 2024-11-07T12:37:02,438 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/f87eebcfb7ed49029e962842a39b8201 is 175, key is test_row_0/A:col10/1730983019598/Put/seqid=0 2024-11-07T12:37:02,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742092_1268 (size=31255) 2024-11-07T12:37:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:37:02,846 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=486, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/f87eebcfb7ed49029e962842a39b8201 2024-11-07T12:37:02,853 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/a10644f932464e8abed68c80196c805d is 50, key is test_row_0/B:col10/1730983019598/Put/seqid=0 2024-11-07T12:37:02,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742093_1269 (size=12301) 2024-11-07T12:37:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:37:03,258 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/a10644f932464e8abed68c80196c805d 2024-11-07T12:37:03,266 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/438161c85ddd43f8a22f03ef77230e5d is 50, key is test_row_0/C:col10/1730983019598/Put/seqid=0 2024-11-07T12:37:03,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742094_1270 (size=12301) 2024-11-07T12:37:03,670 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/438161c85ddd43f8a22f03ef77230e5d 2024-11-07T12:37:03,675 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/A/f87eebcfb7ed49029e962842a39b8201 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f87eebcfb7ed49029e962842a39b8201 2024-11-07T12:37:03,678 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f87eebcfb7ed49029e962842a39b8201, entries=150, sequenceid=486, filesize=30.5 K 2024-11-07T12:37:03,679 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/B/a10644f932464e8abed68c80196c805d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a10644f932464e8abed68c80196c805d 2024-11-07T12:37:03,682 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a10644f932464e8abed68c80196c805d, entries=150, sequenceid=486, filesize=12.0 K 2024-11-07T12:37:03,683 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/.tmp/C/438161c85ddd43f8a22f03ef77230e5d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/438161c85ddd43f8a22f03ef77230e5d 2024-11-07T12:37:03,686 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/438161c85ddd43f8a22f03ef77230e5d, entries=150, sequenceid=486, filesize=12.0 K 2024-11-07T12:37:03,687 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 292f80e0c9fe1beaa809aba371c91f47 in 1666ms, sequenceid=486, compaction requested=true 2024-11-07T12:37:03,687 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f076e2dca36c4c238fd7e445818f2afc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/e843ad3458984638a2d63f8717422d4a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/1c9b49c3d3be4200a5f7825fff0b5550, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b8e710a250d443c68f5d4c63f478d7df, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2fb11a4a06414b27a692ec8551ba865a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/624984436f004d469565af9fa8eac5c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2b84fc0e6d384d70991b8bf7da53c620, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/c31d7a5a4a2945a99763e4ef9b0e9e8b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/82c87bd11ab145c496e48e80c3b0f0c8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/5fe17f06d1144a7b98c7f8d74486e333, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/88f9a59f71c84171b3f75a4a599ebab1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/153e9c2b3fb0434b877688978000edc1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/374ee9574b05482dbabb5b97424d041c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/a8ea5296cd8e4acfb12f6aa21115a168, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/092fb86b262547e68c9de852fb17401f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2da586f3ce744a448023fe98f102a46a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/331932d3208046e4955e8f9b2db1dc84, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d3ce94d7400c427299e031416255c707, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/30776a4d3e6a42c0ac24ed873a067ba1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9521ed7dbb754de694334530a10596ed, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/0a47065fe064408f965d3392ddcd3eee, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b5b30d9daf02405da1090226c33d113e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9f9b7b9d261b4cd9bf52528994c2279a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/ec83a0271e93425dae27520e605d7eae, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/6bb45eb9d4364844a3164892302b2841, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/dcb87f3f818e405c8eaeb0a3065fa9b7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/bbdf0d5b69f348d5bc031ab5895b2b4c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d9fbbafb4d41448abe1d069141388e0f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/908d72ad724c4d68899f31c8b326e3ff, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/cf556e7e29d943f193df6b76b8613d3e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/86768a13eb3a491f91b44f4d8fa345d3] to archive 2024-11-07T12:37:03,688 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:03,689 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f076e2dca36c4c238fd7e445818f2afc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f076e2dca36c4c238fd7e445818f2afc 2024-11-07T12:37:03,690 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/e843ad3458984638a2d63f8717422d4a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/e843ad3458984638a2d63f8717422d4a 2024-11-07T12:37:03,691 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/1c9b49c3d3be4200a5f7825fff0b5550 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/1c9b49c3d3be4200a5f7825fff0b5550 2024-11-07T12:37:03,693 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b8e710a250d443c68f5d4c63f478d7df to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b8e710a250d443c68f5d4c63f478d7df 2024-11-07T12:37:03,694 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2fb11a4a06414b27a692ec8551ba865a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2fb11a4a06414b27a692ec8551ba865a 2024-11-07T12:37:03,695 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/624984436f004d469565af9fa8eac5c3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/624984436f004d469565af9fa8eac5c3 2024-11-07T12:37:03,696 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2b84fc0e6d384d70991b8bf7da53c620 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2b84fc0e6d384d70991b8bf7da53c620 2024-11-07T12:37:03,697 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/c31d7a5a4a2945a99763e4ef9b0e9e8b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/c31d7a5a4a2945a99763e4ef9b0e9e8b 2024-11-07T12:37:03,698 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/82c87bd11ab145c496e48e80c3b0f0c8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/82c87bd11ab145c496e48e80c3b0f0c8 2024-11-07T12:37:03,699 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/5fe17f06d1144a7b98c7f8d74486e333 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/5fe17f06d1144a7b98c7f8d74486e333 2024-11-07T12:37:03,700 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/88f9a59f71c84171b3f75a4a599ebab1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/88f9a59f71c84171b3f75a4a599ebab1 2024-11-07T12:37:03,701 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/153e9c2b3fb0434b877688978000edc1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/153e9c2b3fb0434b877688978000edc1 2024-11-07T12:37:03,702 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/374ee9574b05482dbabb5b97424d041c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/374ee9574b05482dbabb5b97424d041c 2024-11-07T12:37:03,703 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/a8ea5296cd8e4acfb12f6aa21115a168 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/a8ea5296cd8e4acfb12f6aa21115a168 2024-11-07T12:37:03,704 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/092fb86b262547e68c9de852fb17401f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/092fb86b262547e68c9de852fb17401f 2024-11-07T12:37:03,704 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2da586f3ce744a448023fe98f102a46a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/2da586f3ce744a448023fe98f102a46a 2024-11-07T12:37:03,705 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/331932d3208046e4955e8f9b2db1dc84 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/331932d3208046e4955e8f9b2db1dc84 2024-11-07T12:37:03,707 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d3ce94d7400c427299e031416255c707 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d3ce94d7400c427299e031416255c707 2024-11-07T12:37:03,708 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/30776a4d3e6a42c0ac24ed873a067ba1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/30776a4d3e6a42c0ac24ed873a067ba1 2024-11-07T12:37:03,709 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9521ed7dbb754de694334530a10596ed to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9521ed7dbb754de694334530a10596ed 2024-11-07T12:37:03,710 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/0a47065fe064408f965d3392ddcd3eee to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/0a47065fe064408f965d3392ddcd3eee 2024-11-07T12:37:03,711 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b5b30d9daf02405da1090226c33d113e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/b5b30d9daf02405da1090226c33d113e 2024-11-07T12:37:03,712 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9f9b7b9d261b4cd9bf52528994c2279a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/9f9b7b9d261b4cd9bf52528994c2279a 2024-11-07T12:37:03,713 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/ec83a0271e93425dae27520e605d7eae to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/ec83a0271e93425dae27520e605d7eae 2024-11-07T12:37:03,713 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/6bb45eb9d4364844a3164892302b2841 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/6bb45eb9d4364844a3164892302b2841 2024-11-07T12:37:03,714 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/dcb87f3f818e405c8eaeb0a3065fa9b7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/dcb87f3f818e405c8eaeb0a3065fa9b7 2024-11-07T12:37:03,715 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/bbdf0d5b69f348d5bc031ab5895b2b4c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/bbdf0d5b69f348d5bc031ab5895b2b4c 2024-11-07T12:37:03,716 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d9fbbafb4d41448abe1d069141388e0f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/d9fbbafb4d41448abe1d069141388e0f 2024-11-07T12:37:03,717 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/908d72ad724c4d68899f31c8b326e3ff to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/908d72ad724c4d68899f31c8b326e3ff 2024-11-07T12:37:03,718 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/cf556e7e29d943f193df6b76b8613d3e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/cf556e7e29d943f193df6b76b8613d3e 2024-11-07T12:37:03,719 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/86768a13eb3a491f91b44f4d8fa345d3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/86768a13eb3a491f91b44f4d8fa345d3 2024-11-07T12:37:03,720 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/8dff7aab42bd4ba1bc6b26679561559e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/ae714a52dfec4c03bd9b21e3879b074b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/48c3e658bc9b448592f714b8587e2862, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/556c5c5a78444f79a237b41b3f144988, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/76c0121a0bdd433a8a556da621769ee2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/044e5356cc4242d3a92c28bde9510141, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b0cbb8b7d4114d2798f6fa74dbd1d13d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f7fdabed82704a4aa2e18b35ffbe20d4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/e7febe3adc2a4d0da6816022933772c8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/dbb0b5fae8174b82882536f7130d46e0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7fd23ce7de5a490087dc6acb2f3e6e03, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7dc34ee739b94f64b7b4eb412f53e02e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b5e2519af5204723a8c32ceb291b8ef1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/09220d81f3fc4b549eb08997fc8700c1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc14167aabed415d97a867aed6a930fc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c4e03ff14a5e4a5493ab9f7e292055e7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/d1d7be02dce941cd903a293a28a19d89, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/45f4ae9550e141319980212f15422d45, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c56b771a995b4d4ca60d9fae26e8bd42, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f2d3e94d65774253860fd221645e52db, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b325b5f1792e4314b0e70322a355a8d2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/3ca192b09d2c47b2b40e987aab60a50a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc33aa42b4d448999c0d38dfbdff9afe, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/89f45debf1884b239e783c7a402875bb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a176ae0a8ca74fcf9ae28169009e4782, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/6e89a9dad03a4f67b6216f6094d32c75, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c8fcaab780de44368febc9809807861f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/30ddf831e1a94c53b4c76447aec6dfce, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/14fdf7ebd173482a9eb54079b4dedb61, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/2a116937a3c94751a251440ab0488927, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f3877883f727456da61c345f5df50398, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/55e01785e841424ea40728477f8e6e47] to archive 2024-11-07T12:37:03,721 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:03,722 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/8dff7aab42bd4ba1bc6b26679561559e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/8dff7aab42bd4ba1bc6b26679561559e 2024-11-07T12:37:03,723 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/ae714a52dfec4c03bd9b21e3879b074b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/ae714a52dfec4c03bd9b21e3879b074b 2024-11-07T12:37:03,724 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/48c3e658bc9b448592f714b8587e2862 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/48c3e658bc9b448592f714b8587e2862 2024-11-07T12:37:03,725 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/556c5c5a78444f79a237b41b3f144988 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/556c5c5a78444f79a237b41b3f144988 2024-11-07T12:37:03,726 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/76c0121a0bdd433a8a556da621769ee2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/76c0121a0bdd433a8a556da621769ee2 2024-11-07T12:37:03,727 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/044e5356cc4242d3a92c28bde9510141 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/044e5356cc4242d3a92c28bde9510141 2024-11-07T12:37:03,728 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b0cbb8b7d4114d2798f6fa74dbd1d13d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b0cbb8b7d4114d2798f6fa74dbd1d13d 2024-11-07T12:37:03,729 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f7fdabed82704a4aa2e18b35ffbe20d4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f7fdabed82704a4aa2e18b35ffbe20d4 2024-11-07T12:37:03,730 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/e7febe3adc2a4d0da6816022933772c8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/e7febe3adc2a4d0da6816022933772c8 2024-11-07T12:37:03,731 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/dbb0b5fae8174b82882536f7130d46e0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/dbb0b5fae8174b82882536f7130d46e0 2024-11-07T12:37:03,732 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7fd23ce7de5a490087dc6acb2f3e6e03 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7fd23ce7de5a490087dc6acb2f3e6e03 2024-11-07T12:37:03,733 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7dc34ee739b94f64b7b4eb412f53e02e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/7dc34ee739b94f64b7b4eb412f53e02e 2024-11-07T12:37:03,734 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b5e2519af5204723a8c32ceb291b8ef1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b5e2519af5204723a8c32ceb291b8ef1 2024-11-07T12:37:03,735 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/09220d81f3fc4b549eb08997fc8700c1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/09220d81f3fc4b549eb08997fc8700c1 2024-11-07T12:37:03,736 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc14167aabed415d97a867aed6a930fc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc14167aabed415d97a867aed6a930fc 2024-11-07T12:37:03,737 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c4e03ff14a5e4a5493ab9f7e292055e7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c4e03ff14a5e4a5493ab9f7e292055e7 2024-11-07T12:37:03,737 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/d1d7be02dce941cd903a293a28a19d89 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/d1d7be02dce941cd903a293a28a19d89 2024-11-07T12:37:03,738 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/45f4ae9550e141319980212f15422d45 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/45f4ae9550e141319980212f15422d45 2024-11-07T12:37:03,739 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c56b771a995b4d4ca60d9fae26e8bd42 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c56b771a995b4d4ca60d9fae26e8bd42 2024-11-07T12:37:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f2d3e94d65774253860fd221645e52db to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f2d3e94d65774253860fd221645e52db 2024-11-07T12:37:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b325b5f1792e4314b0e70322a355a8d2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/b325b5f1792e4314b0e70322a355a8d2 2024-11-07T12:37:03,742 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/3ca192b09d2c47b2b40e987aab60a50a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/3ca192b09d2c47b2b40e987aab60a50a 2024-11-07T12:37:03,743 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc33aa42b4d448999c0d38dfbdff9afe to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/bc33aa42b4d448999c0d38dfbdff9afe 2024-11-07T12:37:03,744 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/89f45debf1884b239e783c7a402875bb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/89f45debf1884b239e783c7a402875bb 2024-11-07T12:37:03,745 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a176ae0a8ca74fcf9ae28169009e4782 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a176ae0a8ca74fcf9ae28169009e4782 2024-11-07T12:37:03,745 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/6e89a9dad03a4f67b6216f6094d32c75 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/6e89a9dad03a4f67b6216f6094d32c75 2024-11-07T12:37:03,746 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c8fcaab780de44368febc9809807861f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/c8fcaab780de44368febc9809807861f 2024-11-07T12:37:03,747 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/30ddf831e1a94c53b4c76447aec6dfce to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/30ddf831e1a94c53b4c76447aec6dfce 2024-11-07T12:37:03,748 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/14fdf7ebd173482a9eb54079b4dedb61 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/14fdf7ebd173482a9eb54079b4dedb61 2024-11-07T12:37:03,749 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/2a116937a3c94751a251440ab0488927 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/2a116937a3c94751a251440ab0488927 2024-11-07T12:37:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f3877883f727456da61c345f5df50398 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/f3877883f727456da61c345f5df50398 2024-11-07T12:37:03,751 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/55e01785e841424ea40728477f8e6e47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/55e01785e841424ea40728477f8e6e47 2024-11-07T12:37:03,752 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/9f0e2958110142198d25d8f175c0b292, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/126f288507834ae5b2868a7821c0cce9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/dd80b8eade4345e283c82661bc8d638b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/84690929296e4569abe25a4d7d8bd550, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/d31a05a9eda341a8a46dc69952c7a8eb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f3fa9151b5884a0e8f4481cceb6772fb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/702df6069cc549e8b77eb3bd43ce5c5c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/b36831cc46994d84a9b7defd27bf97a5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19d7106242804e32b9254264abe48421, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a9910a2748b34fa78cc372a0644eed88, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65fd4181d6db4f26bc118d510d5403de, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/4a5e889597bf4404a3907b7243eb67e4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/caba2b6d0e95405d8bbd4f37cef34831, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65506d4a33dd4de0ba21ee1145ae7ad5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f0d511650640424dae9df279dcbbab6e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/83c062cb2f2347f082c6deee0c43aedd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/fafafac1b8af4388b5e9e900f7c00e0f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f93d0b0486d24a78921cc2b1dd801b18, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/11bd3d4b1bc847309fddc44531e3385e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a1cab7e52d01473e9013b4358d107f36, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/2ffc4f53cbab4b469adcf61191bfe71e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/102e2e2939bd431b84a18917410e2fe7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/81f959de7a244be3aa8ead35cd72cab2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/aec0672ce8ce484bbcd1f84d77a7615e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/af02ae42eddb4f0b81e90e1ce5c624a6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/eae9d9f0253a4b43a00bd580b85db81c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/ab6928f142964c3a9076e927e03bfa83, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/05ea53d58227466a8ca729ec82414337, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0ac61bd424a544a49d905fb7476c2216, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a867640cc2e94a26b588f9bdf9aa97cf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0455e59e4bf7432e9f3be387e5fe3eac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a6c91732bf5249f7b7647a2fd1185944] to archive 2024-11-07T12:37:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:03,754 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/9f0e2958110142198d25d8f175c0b292 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/9f0e2958110142198d25d8f175c0b292 2024-11-07T12:37:03,755 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/126f288507834ae5b2868a7821c0cce9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/126f288507834ae5b2868a7821c0cce9 2024-11-07T12:37:03,756 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/dd80b8eade4345e283c82661bc8d638b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/dd80b8eade4345e283c82661bc8d638b 2024-11-07T12:37:03,757 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/84690929296e4569abe25a4d7d8bd550 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/84690929296e4569abe25a4d7d8bd550 2024-11-07T12:37:03,758 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/d31a05a9eda341a8a46dc69952c7a8eb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/d31a05a9eda341a8a46dc69952c7a8eb 2024-11-07T12:37:03,758 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f3fa9151b5884a0e8f4481cceb6772fb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f3fa9151b5884a0e8f4481cceb6772fb 2024-11-07T12:37:03,760 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/702df6069cc549e8b77eb3bd43ce5c5c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/702df6069cc549e8b77eb3bd43ce5c5c 2024-11-07T12:37:03,761 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/b36831cc46994d84a9b7defd27bf97a5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/b36831cc46994d84a9b7defd27bf97a5 2024-11-07T12:37:03,761 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19d7106242804e32b9254264abe48421 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19d7106242804e32b9254264abe48421 2024-11-07T12:37:03,762 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a9910a2748b34fa78cc372a0644eed88 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a9910a2748b34fa78cc372a0644eed88 2024-11-07T12:37:03,763 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65fd4181d6db4f26bc118d510d5403de to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65fd4181d6db4f26bc118d510d5403de 2024-11-07T12:37:03,764 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/4a5e889597bf4404a3907b7243eb67e4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/4a5e889597bf4404a3907b7243eb67e4 2024-11-07T12:37:03,765 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/caba2b6d0e95405d8bbd4f37cef34831 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/caba2b6d0e95405d8bbd4f37cef34831 2024-11-07T12:37:03,766 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65506d4a33dd4de0ba21ee1145ae7ad5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/65506d4a33dd4de0ba21ee1145ae7ad5 2024-11-07T12:37:03,767 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f0d511650640424dae9df279dcbbab6e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f0d511650640424dae9df279dcbbab6e 2024-11-07T12:37:03,768 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/83c062cb2f2347f082c6deee0c43aedd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/83c062cb2f2347f082c6deee0c43aedd 2024-11-07T12:37:03,769 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/fafafac1b8af4388b5e9e900f7c00e0f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/fafafac1b8af4388b5e9e900f7c00e0f 2024-11-07T12:37:03,770 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f93d0b0486d24a78921cc2b1dd801b18 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/f93d0b0486d24a78921cc2b1dd801b18 2024-11-07T12:37:03,771 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/11bd3d4b1bc847309fddc44531e3385e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/11bd3d4b1bc847309fddc44531e3385e 2024-11-07T12:37:03,772 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a1cab7e52d01473e9013b4358d107f36 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a1cab7e52d01473e9013b4358d107f36 2024-11-07T12:37:03,772 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/2ffc4f53cbab4b469adcf61191bfe71e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/2ffc4f53cbab4b469adcf61191bfe71e 2024-11-07T12:37:03,773 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/102e2e2939bd431b84a18917410e2fe7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/102e2e2939bd431b84a18917410e2fe7 2024-11-07T12:37:03,774 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/81f959de7a244be3aa8ead35cd72cab2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/81f959de7a244be3aa8ead35cd72cab2 2024-11-07T12:37:03,775 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/aec0672ce8ce484bbcd1f84d77a7615e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/aec0672ce8ce484bbcd1f84d77a7615e 2024-11-07T12:37:03,776 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/af02ae42eddb4f0b81e90e1ce5c624a6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/af02ae42eddb4f0b81e90e1ce5c624a6 2024-11-07T12:37:03,777 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/eae9d9f0253a4b43a00bd580b85db81c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/eae9d9f0253a4b43a00bd580b85db81c 2024-11-07T12:37:03,778 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/ab6928f142964c3a9076e927e03bfa83 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/ab6928f142964c3a9076e927e03bfa83 2024-11-07T12:37:03,779 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/05ea53d58227466a8ca729ec82414337 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/05ea53d58227466a8ca729ec82414337 2024-11-07T12:37:03,780 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0ac61bd424a544a49d905fb7476c2216 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0ac61bd424a544a49d905fb7476c2216 2024-11-07T12:37:03,781 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a867640cc2e94a26b588f9bdf9aa97cf to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a867640cc2e94a26b588f9bdf9aa97cf 2024-11-07T12:37:03,782 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0455e59e4bf7432e9f3be387e5fe3eac to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0455e59e4bf7432e9f3be387e5fe3eac 2024-11-07T12:37:03,783 DEBUG [StoreCloser-TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a6c91732bf5249f7b7647a2fd1185944 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/a6c91732bf5249f7b7647a2fd1185944 2024-11-07T12:37:03,787 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/recovered.edits/489.seqid, newMaxSeqId=489, maxSeqId=4 2024-11-07T12:37:03,787 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. 2024-11-07T12:37:03,788 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 292f80e0c9fe1beaa809aba371c91f47: Waiting for close lock at 1730983022021Running coprocessor pre-close hooks at 1730983022021Disabling compacts and flushes for region at 1730983022021Disabling writes for close at 1730983022021Obtaining lock to block concurrent updates at 1730983022021Preparing flush snapshotting stores in 292f80e0c9fe1beaa809aba371c91f47 at 1730983022021Finished memstore snapshotting TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730983022021Flushing stores of TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47. at 1730983022022 (+1 ms)Flushing 292f80e0c9fe1beaa809aba371c91f47/A: creating writer at 1730983022022Flushing mob file 292f80e0c9fe1beaa809aba371c91f47/A: appending metadata at 1730983022028 (+6 ms)Flushing mob file 292f80e0c9fe1beaa809aba371c91f47/A: closing flushed file at 1730983022028Flushing 292f80e0c9fe1beaa809aba371c91f47/A: appending metadata at 1730983022438 (+410 ms)Flushing 292f80e0c9fe1beaa809aba371c91f47/A: closing flushed file at 1730983022438Flushing 292f80e0c9fe1beaa809aba371c91f47/B: creating writer at 1730983022850 (+412 ms)Flushing 292f80e0c9fe1beaa809aba371c91f47/B: appending metadata at 1730983022853 (+3 ms)Flushing 292f80e0c9fe1beaa809aba371c91f47/B: closing flushed file at 1730983022853Flushing 292f80e0c9fe1beaa809aba371c91f47/C: creating writer at 1730983023262 (+409 ms)Flushing 292f80e0c9fe1beaa809aba371c91f47/C: appending metadata at 1730983023265 (+3 ms)Flushing 292f80e0c9fe1beaa809aba371c91f47/C: closing flushed file at 1730983023265Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29e996bb: reopening flushed file at 1730983023674 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@772caf90: reopening flushed file at 1730983023678 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a1f497e: reopening flushed file at 1730983023682 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 292f80e0c9fe1beaa809aba371c91f47 in 1666ms, sequenceid=486, compaction requested=true at 1730983023687 (+5 ms)Writing region close event to WAL at 1730983023784 (+97 ms)Running coprocessor post-close hooks at 1730983023787 (+3 ms)Closed at 1730983023787 2024-11-07T12:37:03,789 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:03,790 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=292f80e0c9fe1beaa809aba371c91f47, regionState=CLOSED 2024-11-07T12:37:03,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:37:03,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T12:37:03,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 292f80e0c9fe1beaa809aba371c91f47, server=87fb68174c52,38515,1730982961782 in 1.9220 sec 2024-11-07T12:37:03,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T12:37:03,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=292f80e0c9fe1beaa809aba371c91f47, UNASSIGN in 1.9270 sec 2024-11-07T12:37:03,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-07T12:37:03,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9290 sec 2024-11-07T12:37:03,797 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983023797"}]},"ts":"1730983023797"} 2024-11-07T12:37:03,798 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:37:03,798 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:37:03,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9380 sec 2024-11-07T12:37:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:37:04,008 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:04,008 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T12:37:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:04,009 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T12:37:04,010 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:04,013 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,015 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/recovered.edits] 2024-11-07T12:37:04,018 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/7bdee6cba73045279c9f9f7ffa9bf32c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/7bdee6cba73045279c9f9f7ffa9bf32c 2024-11-07T12:37:04,020 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/872ad2cf79704e48b795663e702f68d5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/872ad2cf79704e48b795663e702f68d5 2024-11-07T12:37:04,021 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f87eebcfb7ed49029e962842a39b8201 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/A/f87eebcfb7ed49029e962842a39b8201 2024-11-07T12:37:04,023 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/268cdf0e8ef442578a05016d0b8bcc4c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/268cdf0e8ef442578a05016d0b8bcc4c 2024-11-07T12:37:04,024 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/359bc0a2017a4d0fb20e61d0129497a4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/359bc0a2017a4d0fb20e61d0129497a4 2024-11-07T12:37:04,026 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a10644f932464e8abed68c80196c805d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/B/a10644f932464e8abed68c80196c805d 2024-11-07T12:37:04,028 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0db50381015842ab8518f9d2da84cfea to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/0db50381015842ab8518f9d2da84cfea 2024-11-07T12:37:04,029 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19da44c7899e47c7ac3d12559dcb1018 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/19da44c7899e47c7ac3d12559dcb1018 2024-11-07T12:37:04,030 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/438161c85ddd43f8a22f03ef77230e5d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/C/438161c85ddd43f8a22f03ef77230e5d 2024-11-07T12:37:04,033 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/recovered.edits/489.seqid to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47/recovered.edits/489.seqid 2024-11-07T12:37:04,033 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,033 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:37:04,034 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:37:04,034 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:37:04,037 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110701c3c998e4a74297b048a4afdc6ad2c1_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110701c3c998e4a74297b048a4afdc6ad2c1_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,038 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107022e13e586fa4664a298adba96e3744d_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107022e13e586fa4664a298adba96e3744d_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,039 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110704ee0f227f194fc2ada8a47ce2ec2d67_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110704ee0f227f194fc2ada8a47ce2ec2d67_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,041 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107271a3ae111b448afab93b17497f96ea8_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107271a3ae111b448afab93b17497f96ea8_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,041 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073d399c1ff84344d988c401b8688f0fae_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073d399c1ff84344d988c401b8688f0fae_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,043 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110744bb6bde89064e5a85d850c373bbfce7_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110744bb6bde89064e5a85d850c373bbfce7_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,044 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107457e2054103143f19072b9c0c34662d3_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107457e2054103143f19072b9c0c34662d3_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,045 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110747217eb6f0c748ffb8c7c8870803dd63_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110747217eb6f0c748ffb8c7c8870803dd63_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,046 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075250eb1a93784b7b852909267b5b7ee9_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075250eb1a93784b7b852909267b5b7ee9_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,047 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075347ae6daf9a4ccf983b9850d3b769b9_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075347ae6daf9a4ccf983b9850d3b769b9_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,048 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754e589d38b6a4862a53da86066816cb3_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754e589d38b6a4862a53da86066816cb3_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,049 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075807873811c249e0a864a2dcafb183b2_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075807873811c249e0a864a2dcafb183b2_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,051 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d45a470fa2444928a29ce5938abbf14_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d45a470fa2444928a29ce5938abbf14_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,052 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075dcd91c03b66477fa05f3d061d0c5b0e_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075dcd91c03b66477fa05f3d061d0c5b0e_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,053 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110761dcc37ab9594a86ad945cfe0cc7f549_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110761dcc37ab9594a86ad945cfe0cc7f549_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,054 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077b2114f985e44e3085309adbdd046dd0_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077b2114f985e44e3085309adbdd046dd0_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,055 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107888589217a0c4a169a38dd814eba6f8e_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107888589217a0c4a169a38dd814eba6f8e_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,056 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a4f0d1ca46f4a609fbeabf7103a573e_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a4f0d1ca46f4a609fbeabf7103a573e_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,057 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079b55f13d51b34aea9def067747a33197_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079b55f13d51b34aea9def067747a33197_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,058 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aaa8be2155e943cfbe877b37aecb709e_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aaa8be2155e943cfbe877b37aecb709e_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,059 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b3f021e1a4e242a197261829a041d6bd_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b3f021e1a4e242a197261829a041d6bd_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,060 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d5c435b5978f419682001df38a169347_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d5c435b5978f419682001df38a169347_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,062 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d6ae6d7c6d7a4df39357d9314afe6da7_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d6ae6d7c6d7a4df39357d9314afe6da7_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,063 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f59870844a944a1482d10cd5243b08ad_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f59870844a944a1482d10cd5243b08ad_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f75ac0a465134db38e14bfba33ecb427_292f80e0c9fe1beaa809aba371c91f47 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f75ac0a465134db38e14bfba33ecb427_292f80e0c9fe1beaa809aba371c91f47 2024-11-07T12:37:04,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:37:04,066 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:04,068 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:37:04,070 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:37:04,071 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:04,071 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:37:04,071 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983024071"}]},"ts":"9223372036854775807"} 2024-11-07T12:37:04,073 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:37:04,073 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 292f80e0c9fe1beaa809aba371c91f47, NAME => 'TestAcidGuarantees,,1730982994680.292f80e0c9fe1beaa809aba371c91f47.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:37:04,073 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:37:04,073 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983024073"}]},"ts":"9223372036854775807"} 2024-11-07T12:37:04,075 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:37:04,075 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:04,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-07T12:37:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T12:37:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:37:04,118 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:04,126 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 239) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1848162525_22 at /127.0.0.1:41340 [Waiting for operation #1003] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1848162525_22 at /127.0.0.1:41806 [Waiting for operation #841] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1590488155_22 at /127.0.0.1:37114 [Waiting for operation #1249] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1590488155_22 at /127.0.0.1:52776 [Waiting for operation #323] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=475 (was 465) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7270 (was 6096) - AvailableMemoryMB LEAK? - 2024-11-07T12:37:04,135 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=475, ProcessCount=11, AvailableMemoryMB=7270 2024-11-07T12:37:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:37:04,136 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:37:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:04,138 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:37:04,138 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:04,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-07T12:37:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:37:04,139 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:37:04,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742095_1271 (size=959) 2024-11-07T12:37:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:37:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:37:04,546 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e 2024-11-07T12:37:04,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742096_1272 (size=53) 2024-11-07T12:37:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:37:04,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:37:04,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bf2b7ddfed68e71c2777c670069c266b, disabling compactions & flushes 2024-11-07T12:37:04,952 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:04,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:04,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. after waiting 0 ms 2024-11-07T12:37:04,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:04,952 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:04,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bf2b7ddfed68e71c2777c670069c266b: Waiting for close lock at 1730983024952Disabling compacts and flushes for region at 1730983024952Disabling writes for close at 1730983024952Writing region close event to WAL at 1730983024952Closed at 1730983024952 2024-11-07T12:37:04,953 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:37:04,953 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983024953"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983024953"}]},"ts":"1730983024953"} 2024-11-07T12:37:04,955 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:37:04,955 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:37:04,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983024955"}]},"ts":"1730983024955"} 2024-11-07T12:37:04,956 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:37:04,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, ASSIGN}] 2024-11-07T12:37:04,958 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, ASSIGN 2024-11-07T12:37:04,958 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, ASSIGN; state=OFFLINE, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=false 2024-11-07T12:37:05,109 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=bf2b7ddfed68e71c2777c670069c266b, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:05,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, ASSIGN because future has completed 2024-11-07T12:37:05,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:37:05,265 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:05,265 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:37:05,265 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,265 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:37:05,265 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,265 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,266 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,268 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf2b7ddfed68e71c2777c670069c266b columnFamilyName A 2024-11-07T12:37:05,268 DEBUG [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:05,268 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] regionserver.HStore(327): Store=bf2b7ddfed68e71c2777c670069c266b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:05,268 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,269 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf2b7ddfed68e71c2777c670069c266b columnFamilyName B 2024-11-07T12:37:05,269 DEBUG [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:05,270 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] regionserver.HStore(327): Store=bf2b7ddfed68e71c2777c670069c266b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:05,270 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,271 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf2b7ddfed68e71c2777c670069c266b columnFamilyName C 2024-11-07T12:37:05,271 DEBUG [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:05,271 INFO [StoreOpener-bf2b7ddfed68e71c2777c670069c266b-1 {}] regionserver.HStore(327): Store=bf2b7ddfed68e71c2777c670069c266b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:05,271 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,272 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,272 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,273 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,273 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,273 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:37:05,274 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,277 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:37:05,277 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened bf2b7ddfed68e71c2777c670069c266b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67359623, jitterRate=0.0037366002798080444}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:37:05,277 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:37:05,278 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for bf2b7ddfed68e71c2777c670069c266b: Running coprocessor pre-open hook at 1730983025265Writing region info on filesystem at 1730983025265Initializing all the Stores at 1730983025266 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983025266Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983025266Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983025266Cleaning up temporary data from old regions at 1730983025273 (+7 ms)Running coprocessor post-open hooks at 1730983025277 (+4 ms)Region opened successfully at 1730983025278 (+1 ms) 2024-11-07T12:37:05,279 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., pid=63, masterSystemTime=1730983025262 2024-11-07T12:37:05,280 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:05,280 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:05,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=bf2b7ddfed68e71c2777c670069c266b, regionState=OPEN, openSeqNum=2, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:05,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:37:05,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-07T12:37:05,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 in 172 msec 2024-11-07T12:37:05,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T12:37:05,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, ASSIGN in 327 msec 2024-11-07T12:37:05,286 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:37:05,286 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983025286"}]},"ts":"1730983025286"} 2024-11-07T12:37:05,287 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:37:05,288 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:37:05,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-07T12:37:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:37:06,288 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:06,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,290 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc72d7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,292 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4083a33c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,292 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:37:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb68174c52,36251,1730982961115 2024-11-07T12:37:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a60a224 2024-11-07T12:37:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:37:06,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:37:06,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,298 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4eeed4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6743077c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,301 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c633e58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5191f12e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,304 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c6cccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,305 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ce936e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,306 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7edf3a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,308 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf1b05d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,310 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1964d6c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,311 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b891b5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,312 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11818ec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,313 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3f962a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,315 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e1c259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626625d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,318 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bcfba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,319 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@493fe7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,322 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66fd06da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,323 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8af25d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:06,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:06,325 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca8a0f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:06,326 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@535ebe36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:06,327 DEBUG [Thread-1248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,327 DEBUG [Thread-1263 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,327 DEBUG [Thread-1252 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,328 DEBUG [Thread-1257 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,328 DEBUG [Thread-1261 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,328 DEBUG [Thread-1254 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,330 DEBUG [Thread-1265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,330 DEBUG [Thread-1259 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,331 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:06,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-07T12:37:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:37:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,335 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,336 DEBUG [Thread-1246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,336 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:06,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,347 DEBUG [Thread-1250 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:06,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:06,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:06,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:37:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:37:06,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/c1be008d93cf44d688a83bc8116aee17 is 50, key is test_row_0/A:col10/1730983026349/Put/seqid=0 2024-11-07T12:37:06,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742097_1273 (size=12001) 2024-11-07T12:37:06,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983086383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983086384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/c1be008d93cf44d688a83bc8116aee17 2024-11-07T12:37:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983086389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983086392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983086392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/941dedf33fb344688d53e2740422954f is 50, key is test_row_0/B:col10/1730983026349/Put/seqid=0 2024-11-07T12:37:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:37:06,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742098_1274 (size=12001) 2024-11-07T12:37:06,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/941dedf33fb344688d53e2740422954f 2024-11-07T12:37:06,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/41e5b5bae1394229b005ee541999d06f is 50, key is test_row_0/C:col10/1730983026349/Put/seqid=0 2024-11-07T12:37:06,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:37:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:06,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983086497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983086498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983086498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742099_1275 (size=12001) 2024-11-07T12:37:06,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983086507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983086507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:37:06,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:37:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:06,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983086707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983086707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983086707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983086717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983086718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:06,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:37:06,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:06,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:06,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:06,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:06,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/41e5b5bae1394229b005ee541999d06f 2024-11-07T12:37:06,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/c1be008d93cf44d688a83bc8116aee17 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c1be008d93cf44d688a83bc8116aee17 2024-11-07T12:37:06,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c1be008d93cf44d688a83bc8116aee17, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T12:37:06,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/941dedf33fb344688d53e2740422954f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/941dedf33fb344688d53e2740422954f 2024-11-07T12:37:06,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/941dedf33fb344688d53e2740422954f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T12:37:06,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/41e5b5bae1394229b005ee541999d06f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/41e5b5bae1394229b005ee541999d06f 2024-11-07T12:37:06,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/41e5b5bae1394229b005ee541999d06f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T12:37:06,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for bf2b7ddfed68e71c2777c670069c266b in 580ms, sequenceid=14, compaction requested=false 2024-11-07T12:37:06,931 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T12:37:06,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:06,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:37:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:06,956 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:37:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:37:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/3dc474e3eae94642838a651cf51d543e is 50, key is test_row_0/A:col10/1730983026389/Put/seqid=0 2024-11-07T12:37:06,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742100_1276 (size=12001) 2024-11-07T12:37:06,981 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/3dc474e3eae94642838a651cf51d543e 2024-11-07T12:37:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/a203151dcf6a4355aff5fe240994ac4e is 50, key is test_row_0/B:col10/1730983026389/Put/seqid=0 2024-11-07T12:37:07,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742101_1277 (size=12001) 2024-11-07T12:37:07,008 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/a203151dcf6a4355aff5fe240994ac4e 2024-11-07T12:37:07,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:07,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/ff55279ccfac4526b45c4ab5b9a88bdc is 50, key is test_row_0/C:col10/1730983026389/Put/seqid=0 2024-11-07T12:37:07,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742102_1278 (size=12001) 2024-11-07T12:37:07,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983087033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,041 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/ff55279ccfac4526b45c4ab5b9a88bdc 2024-11-07T12:37:07,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983087035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983087035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983087041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983087042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/3dc474e3eae94642838a651cf51d543e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/3dc474e3eae94642838a651cf51d543e 2024-11-07T12:37:07,058 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/3dc474e3eae94642838a651cf51d543e, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:37:07,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/a203151dcf6a4355aff5fe240994ac4e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a203151dcf6a4355aff5fe240994ac4e 2024-11-07T12:37:07,064 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a203151dcf6a4355aff5fe240994ac4e, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:37:07,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/ff55279ccfac4526b45c4ab5b9a88bdc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/ff55279ccfac4526b45c4ab5b9a88bdc 2024-11-07T12:37:07,074 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/ff55279ccfac4526b45c4ab5b9a88bdc, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:37:07,074 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for bf2b7ddfed68e71c2777c670069c266b in 118ms, sequenceid=38, compaction requested=false 2024-11-07T12:37:07,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:07,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:07,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-07T12:37:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-07T12:37:07,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T12:37:07,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 739 msec 2024-11-07T12:37:07,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 748 msec 2024-11-07T12:37:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:07,150 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:07,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:37:07,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/7c96459552f54b6a9c3bb2fe061615b4 is 50, key is test_row_0/A:col10/1730983027150/Put/seqid=0 2024-11-07T12:37:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742103_1279 (size=16681) 2024-11-07T12:37:07,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983087172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983087172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983087172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983087173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983087174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983087297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983087297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983087297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983087297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983087308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:37:07,467 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:07,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-07T12:37:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:37:07,470 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:07,470 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:07,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:07,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983087507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983087507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983087508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983087508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983087517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/7c96459552f54b6a9c3bb2fe061615b4 2024-11-07T12:37:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:37:07,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/54f5497702f64cb7879e59f165a50732 is 50, key is test_row_0/B:col10/1730983027150/Put/seqid=0 2024-11-07T12:37:07,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742104_1280 (size=12001) 2024-11-07T12:37:07,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:37:07,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:07,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:07,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:07,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:37:07,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:07,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:07,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:07,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:37:07,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983087817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983087818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983087818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983087818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983087827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:07,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:37:07,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:07,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:07,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:07,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:07,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/54f5497702f64cb7879e59f165a50732 2024-11-07T12:37:07,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/508a7128d5264d04adf0f153339db780 is 50, key is test_row_0/C:col10/1730983027150/Put/seqid=0 2024-11-07T12:37:07,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742105_1281 (size=12001) 2024-11-07T12:37:07,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/508a7128d5264d04adf0f153339db780 2024-11-07T12:37:08,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/7c96459552f54b6a9c3bb2fe061615b4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/7c96459552f54b6a9c3bb2fe061615b4 2024-11-07T12:37:08,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/7c96459552f54b6a9c3bb2fe061615b4, entries=250, sequenceid=56, filesize=16.3 K 2024-11-07T12:37:08,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/54f5497702f64cb7879e59f165a50732 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/54f5497702f64cb7879e59f165a50732 2024-11-07T12:37:08,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/54f5497702f64cb7879e59f165a50732, entries=150, sequenceid=56, filesize=11.7 K 2024-11-07T12:37:08,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/508a7128d5264d04adf0f153339db780 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/508a7128d5264d04adf0f153339db780 2024-11-07T12:37:08,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/508a7128d5264d04adf0f153339db780, entries=150, sequenceid=56, filesize=11.7 K 2024-11-07T12:37:08,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for bf2b7ddfed68e71c2777c670069c266b in 866ms, sequenceid=56, compaction requested=true 2024-11-07T12:37:08,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:08,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:08,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:08,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:08,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:08,017 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:08,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:08,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:08,017 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:08,018 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:08,018 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:08,018 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:08,018 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c1be008d93cf44d688a83bc8116aee17, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/3dc474e3eae94642838a651cf51d543e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/7c96459552f54b6a9c3bb2fe061615b4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=39.7 K 2024-11-07T12:37:08,018 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:08,018 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1be008d93cf44d688a83bc8116aee17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730983026345 2024-11-07T12:37:08,018 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:08,019 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:08,019 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/941dedf33fb344688d53e2740422954f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a203151dcf6a4355aff5fe240994ac4e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/54f5497702f64cb7879e59f165a50732] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=35.2 K 2024-11-07T12:37:08,020 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 941dedf33fb344688d53e2740422954f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730983026345 2024-11-07T12:37:08,020 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc474e3eae94642838a651cf51d543e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730983026380 2024-11-07T12:37:08,020 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c96459552f54b6a9c3bb2fe061615b4, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983027149 2024-11-07T12:37:08,021 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a203151dcf6a4355aff5fe240994ac4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730983026380 2024-11-07T12:37:08,021 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f5497702f64cb7879e59f165a50732, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983027149 2024-11-07T12:37:08,031 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:08,032 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4d6a4d6997ae43b29880652d2b69db36 is 50, key is test_row_0/A:col10/1730983027150/Put/seqid=0 2024-11-07T12:37:08,050 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:08,051 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/a965b9733276463f8e209e1c4793ca2e is 50, key is test_row_0/B:col10/1730983027150/Put/seqid=0 2024-11-07T12:37:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742106_1282 (size=12104) 2024-11-07T12:37:08,059 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4d6a4d6997ae43b29880652d2b69db36 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4d6a4d6997ae43b29880652d2b69db36 2024-11-07T12:37:08,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742107_1283 (size=12104) 2024-11-07T12:37:08,064 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into 4d6a4d6997ae43b29880652d2b69db36(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:08,064 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:08,064 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=13, startTime=1730983028016; duration=0sec 2024-11-07T12:37:08,065 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:08,065 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:08,065 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:08,066 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:08,066 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/C is initiating minor compaction (all files) 2024-11-07T12:37:08,066 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/C in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:08,066 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/41e5b5bae1394229b005ee541999d06f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/ff55279ccfac4526b45c4ab5b9a88bdc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/508a7128d5264d04adf0f153339db780] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=35.2 K 2024-11-07T12:37:08,067 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41e5b5bae1394229b005ee541999d06f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730983026345 2024-11-07T12:37:08,068 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff55279ccfac4526b45c4ab5b9a88bdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730983026380 2024-11-07T12:37:08,068 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 508a7128d5264d04adf0f153339db780, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983027149 2024-11-07T12:37:08,071 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/a965b9733276463f8e209e1c4793ca2e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a965b9733276463f8e209e1c4793ca2e 2024-11-07T12:37:08,080 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into a965b9733276463f8e209e1c4793ca2e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:08,080 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:08,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:37:08,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:08,082 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T12:37:08,082 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=13, startTime=1730983028017; duration=0sec 2024-11-07T12:37:08,082 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:08,082 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:08,082 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:08,083 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/b173b1c5995c415c935df84c6e2a475e is 50, key is test_row_0/C:col10/1730983027150/Put/seqid=0 2024-11-07T12:37:08,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/1e55d2f229e741c0845fdde6ded60b25 is 50, key is test_row_0/A:col10/1730983027170/Put/seqid=0 2024-11-07T12:37:08,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742108_1284 (size=12104) 2024-11-07T12:37:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:37:08,104 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/b173b1c5995c415c935df84c6e2a475e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/b173b1c5995c415c935df84c6e2a475e 2024-11-07T12:37:08,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742109_1285 (size=12001) 2024-11-07T12:37:08,108 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/1e55d2f229e741c0845fdde6ded60b25 2024-11-07T12:37:08,113 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/C of bf2b7ddfed68e71c2777c670069c266b into b173b1c5995c415c935df84c6e2a475e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:08,113 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:08,113 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/C, priority=13, startTime=1730983028017; duration=0sec 2024-11-07T12:37:08,113 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:08,113 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:08,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/2026b3aca09a4dd3b9413ba9df9409da is 50, key is test_row_0/B:col10/1730983027170/Put/seqid=0 2024-11-07T12:37:08,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742110_1286 (size=12001) 2024-11-07T12:37:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:08,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:08,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983088341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983088342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983088343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983088343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983088345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983088458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983088458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983088458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983088458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983088458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,531 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/2026b3aca09a4dd3b9413ba9df9409da 2024-11-07T12:37:08,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/6101172039754d7dae781b4e82aa5434 is 50, key is test_row_0/C:col10/1730983027170/Put/seqid=0 2024-11-07T12:37:08,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742111_1287 (size=12001) 2024-11-07T12:37:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:37:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983088668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983088668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983088668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983088668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983088668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:08,944 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/6101172039754d7dae781b4e82aa5434 2024-11-07T12:37:08,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/1e55d2f229e741c0845fdde6ded60b25 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/1e55d2f229e741c0845fdde6ded60b25 2024-11-07T12:37:08,952 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/1e55d2f229e741c0845fdde6ded60b25, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T12:37:08,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/2026b3aca09a4dd3b9413ba9df9409da as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/2026b3aca09a4dd3b9413ba9df9409da 2024-11-07T12:37:08,957 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/2026b3aca09a4dd3b9413ba9df9409da, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T12:37:08,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/6101172039754d7dae781b4e82aa5434 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/6101172039754d7dae781b4e82aa5434 2024-11-07T12:37:08,961 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/6101172039754d7dae781b4e82aa5434, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T12:37:08,962 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for bf2b7ddfed68e71c2777c670069c266b in 880ms, sequenceid=76, compaction requested=false 2024-11-07T12:37:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-07T12:37:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-07T12:37:08,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T12:37:08,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4920 sec 2024-11-07T12:37:08,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4980 sec 2024-11-07T12:37:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:08,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:08,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T12:37:08,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/198074e8683d4ab7921161c5fc7f4409 is 50, key is test_row_0/A:col10/1730983028344/Put/seqid=0 2024-11-07T12:37:08,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742112_1288 (size=16681) 2024-11-07T12:37:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983088994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983088994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983088995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983088996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983088997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983089107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983089107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983089107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983089107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983089107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,312 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:37:09,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983089317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983089317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983089318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983089318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983089318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/198074e8683d4ab7921161c5fc7f4409 2024-11-07T12:37:09,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/dfe58b01896d4affa1c680dea6a77514 is 50, key is test_row_0/B:col10/1730983028344/Put/seqid=0 2024-11-07T12:37:09,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742113_1289 (size=12001) 2024-11-07T12:37:09,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/dfe58b01896d4affa1c680dea6a77514 2024-11-07T12:37:09,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/fa4b0d6d33a14e648b7c1f29e50d15cc is 50, key is test_row_0/C:col10/1730983028344/Put/seqid=0 2024-11-07T12:37:09,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742114_1290 (size=12001) 2024-11-07T12:37:09,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/fa4b0d6d33a14e648b7c1f29e50d15cc 2024-11-07T12:37:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/198074e8683d4ab7921161c5fc7f4409 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/198074e8683d4ab7921161c5fc7f4409 2024-11-07T12:37:09,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/198074e8683d4ab7921161c5fc7f4409, entries=250, sequenceid=98, filesize=16.3 K 2024-11-07T12:37:09,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/dfe58b01896d4affa1c680dea6a77514 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/dfe58b01896d4affa1c680dea6a77514 2024-11-07T12:37:09,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/dfe58b01896d4affa1c680dea6a77514, entries=150, sequenceid=98, filesize=11.7 K 2024-11-07T12:37:09,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/fa4b0d6d33a14e648b7c1f29e50d15cc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa4b0d6d33a14e648b7c1f29e50d15cc 2024-11-07T12:37:09,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa4b0d6d33a14e648b7c1f29e50d15cc, entries=150, sequenceid=98, filesize=11.7 K 2024-11-07T12:37:09,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for bf2b7ddfed68e71c2777c670069c266b in 471ms, sequenceid=98, compaction requested=true 2024-11-07T12:37:09,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:09,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:09,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:09,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:09,451 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:09,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:09,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:09,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:09,452 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:09,453 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:09,453 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:09,453 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:09,453 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a965b9733276463f8e209e1c4793ca2e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/2026b3aca09a4dd3b9413ba9df9409da, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/dfe58b01896d4affa1c680dea6a77514] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=35.3 K 2024-11-07T12:37:09,453 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:09,453 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:09,453 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:09,453 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4d6a4d6997ae43b29880652d2b69db36, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/1e55d2f229e741c0845fdde6ded60b25, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/198074e8683d4ab7921161c5fc7f4409] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=39.8 K 2024-11-07T12:37:09,454 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d6a4d6997ae43b29880652d2b69db36, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983027149 2024-11-07T12:37:09,454 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a965b9733276463f8e209e1c4793ca2e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983027149 2024-11-07T12:37:09,456 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e55d2f229e741c0845fdde6ded60b25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730983027170 2024-11-07T12:37:09,456 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2026b3aca09a4dd3b9413ba9df9409da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730983027170 2024-11-07T12:37:09,456 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 198074e8683d4ab7921161c5fc7f4409, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730983028341 2024-11-07T12:37:09,456 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe58b01896d4affa1c680dea6a77514, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730983028344 2024-11-07T12:37:09,466 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:09,466 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/0cbbf4b5ea2b4f6c944724761c7ee8fb is 50, key is test_row_0/B:col10/1730983028344/Put/seqid=0 2024-11-07T12:37:09,467 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:09,467 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/e0f332badef04ac2957e35d77fa8d298 is 50, key is test_row_0/A:col10/1730983028344/Put/seqid=0 2024-11-07T12:37:09,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742115_1291 (size=12207) 2024-11-07T12:37:09,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742116_1292 (size=12207) 2024-11-07T12:37:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:37:09,617 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:09,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-07T12:37:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:37:09,619 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:09,620 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:09,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:09,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T12:37:09,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/af204dc360c14597b0b5bfded1a74177 is 50, key is test_row_0/A:col10/1730983028995/Put/seqid=0 2024-11-07T12:37:09,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742117_1293 (size=12001) 2024-11-07T12:37:09,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983089648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983089649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983089649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983089650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983089653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:37:09,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983089758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983089758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983089758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983089758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983089767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:09,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:09,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:09,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:09,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:09,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:09,883 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/0cbbf4b5ea2b4f6c944724761c7ee8fb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cbbf4b5ea2b4f6c944724761c7ee8fb 2024-11-07T12:37:09,886 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/e0f332badef04ac2957e35d77fa8d298 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/e0f332badef04ac2957e35d77fa8d298 2024-11-07T12:37:09,888 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into 0cbbf4b5ea2b4f6c944724761c7ee8fb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:09,888 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:09,888 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=13, startTime=1730983029451; duration=0sec 2024-11-07T12:37:09,888 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:09,888 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:09,888 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:09,889 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:09,889 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/C is initiating minor compaction (all files) 2024-11-07T12:37:09,890 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/C in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:09,890 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/b173b1c5995c415c935df84c6e2a475e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/6101172039754d7dae781b4e82aa5434, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa4b0d6d33a14e648b7c1f29e50d15cc] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=35.3 K 2024-11-07T12:37:09,890 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b173b1c5995c415c935df84c6e2a475e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983027149 2024-11-07T12:37:09,891 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into e0f332badef04ac2957e35d77fa8d298(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:09,891 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:09,891 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=13, startTime=1730983029451; duration=0sec 2024-11-07T12:37:09,891 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:09,891 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:09,891 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 6101172039754d7dae781b4e82aa5434, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730983027170 2024-11-07T12:37:09,891 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4b0d6d33a14e648b7c1f29e50d15cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730983028344 2024-11-07T12:37:09,898 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:09,899 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/115decc5bed54b09bc716a22e8c91481 is 50, key is test_row_0/C:col10/1730983028344/Put/seqid=0 2024-11-07T12:37:09,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742118_1294 (size=12207) 2024-11-07T12:37:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:37:09,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:09,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:09,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983089967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983089967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983089967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983089968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:09,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983089977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:09,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:09,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/af204dc360c14597b0b5bfded1a74177 2024-11-07T12:37:10,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/d0a88976344949da9c72302447f6b9d2 is 50, key is test_row_0/B:col10/1730983028995/Put/seqid=0 2024-11-07T12:37:10,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742119_1295 (size=12001) 2024-11-07T12:37:10,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:37:10,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983090277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983090277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983090277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983090277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983090287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,309 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/115decc5bed54b09bc716a22e8c91481 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/115decc5bed54b09bc716a22e8c91481 2024-11-07T12:37:10,314 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/C of bf2b7ddfed68e71c2777c670069c266b into 115decc5bed54b09bc716a22e8c91481(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:10,314 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:10,314 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/C, priority=13, startTime=1730983029451; duration=0sec 2024-11-07T12:37:10,315 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:10,315 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:10,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/d0a88976344949da9c72302447f6b9d2 2024-11-07T12:37:10,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/0a318d5a6af8456e9ac1143626fd577f is 50, key is test_row_0/C:col10/1730983028995/Put/seqid=0 2024-11-07T12:37:10,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742120_1296 (size=12001) 2024-11-07T12:37:10,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:37:10,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983090787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983090787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983090787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983090787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983090797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:10,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:10,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:10,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/0a318d5a6af8456e9ac1143626fd577f 2024-11-07T12:37:10,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/af204dc360c14597b0b5bfded1a74177 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/af204dc360c14597b0b5bfded1a74177 2024-11-07T12:37:10,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/af204dc360c14597b0b5bfded1a74177, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T12:37:10,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/d0a88976344949da9c72302447f6b9d2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/d0a88976344949da9c72302447f6b9d2 2024-11-07T12:37:10,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/d0a88976344949da9c72302447f6b9d2, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T12:37:10,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/0a318d5a6af8456e9ac1143626fd577f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/0a318d5a6af8456e9ac1143626fd577f 2024-11-07T12:37:10,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/0a318d5a6af8456e9ac1143626fd577f, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T12:37:10,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for bf2b7ddfed68e71c2777c670069c266b in 1262ms, sequenceid=115, compaction requested=false 2024-11-07T12:37:10,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:11,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:37:11,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:11,011 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T12:37:11,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/b86e81cff75147f98b881fa32f96a86d is 50, key is test_row_0/A:col10/1730983029648/Put/seqid=0 2024-11-07T12:37:11,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742121_1297 (size=12101) 2024-11-07T12:37:11,021 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/b86e81cff75147f98b881fa32f96a86d 2024-11-07T12:37:11,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/0cf518931332421cbd71564043aec810 is 50, key is test_row_0/B:col10/1730983029648/Put/seqid=0 2024-11-07T12:37:11,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742122_1298 (size=12101) 2024-11-07T12:37:11,432 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/0cf518931332421cbd71564043aec810 2024-11-07T12:37:11,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/12aedffd71774e7fa7319b3960b3ac49 is 50, key is test_row_0/C:col10/1730983029648/Put/seqid=0 2024-11-07T12:37:11,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742123_1299 (size=12101) 2024-11-07T12:37:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:37:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:11,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983091819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983091820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983091822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983091819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:11,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983091829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,833 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:11,854 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/12aedffd71774e7fa7319b3960b3ac49 2024-11-07T12:37:11,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/b86e81cff75147f98b881fa32f96a86d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/b86e81cff75147f98b881fa32f96a86d 2024-11-07T12:37:11,863 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/b86e81cff75147f98b881fa32f96a86d, entries=150, sequenceid=137, filesize=11.8 K 2024-11-07T12:37:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/0cf518931332421cbd71564043aec810 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cf518931332421cbd71564043aec810 2024-11-07T12:37:11,868 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cf518931332421cbd71564043aec810, entries=150, sequenceid=137, filesize=11.8 K 2024-11-07T12:37:11,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/12aedffd71774e7fa7319b3960b3ac49 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/12aedffd71774e7fa7319b3960b3ac49 2024-11-07T12:37:11,872 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/12aedffd71774e7fa7319b3960b3ac49, entries=150, sequenceid=137, filesize=11.8 K 2024-11-07T12:37:11,873 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for bf2b7ddfed68e71c2777c670069c266b in 862ms, sequenceid=137, compaction requested=true 2024-11-07T12:37:11,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:11,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:11,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-07T12:37:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-07T12:37:11,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-07T12:37:11,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2540 sec 2024-11-07T12:37:11,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.2580 sec 2024-11-07T12:37:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:11,940 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:11,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T12:37:11,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4167feb2adfa4b95a4dd3ddfda159bcd is 50, key is test_row_0/A:col10/1730983031819/Put/seqid=0 2024-11-07T12:37:11,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742124_1300 (size=14541) 2024-11-07T12:37:11,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983091957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983091958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:11,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983091960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:11,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983091962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983092068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983092068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983092068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983092068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983092278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983092278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983092278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983092278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4167feb2adfa4b95a4dd3ddfda159bcd 2024-11-07T12:37:12,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/325ce207fb334e4099e9e37d60bf5316 is 50, key is test_row_0/B:col10/1730983031819/Put/seqid=0 2024-11-07T12:37:12,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742125_1301 (size=12151) 2024-11-07T12:37:12,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983092587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983092588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983092588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983092588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:12,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/325ce207fb334e4099e9e37d60bf5316 2024-11-07T12:37:12,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/e8512e2c5e2e47be819e7b3b5cc24f7a is 50, key is test_row_0/C:col10/1730983031819/Put/seqid=0 2024-11-07T12:37:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742126_1302 (size=12151) 2024-11-07T12:37:13,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983093098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:13,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983093098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:13,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983093098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:13,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:13,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983093107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:13,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/e8512e2c5e2e47be819e7b3b5cc24f7a 2024-11-07T12:37:13,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4167feb2adfa4b95a4dd3ddfda159bcd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4167feb2adfa4b95a4dd3ddfda159bcd 2024-11-07T12:37:13,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4167feb2adfa4b95a4dd3ddfda159bcd, entries=200, sequenceid=155, filesize=14.2 K 2024-11-07T12:37:13,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/325ce207fb334e4099e9e37d60bf5316 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/325ce207fb334e4099e9e37d60bf5316 2024-11-07T12:37:13,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/325ce207fb334e4099e9e37d60bf5316, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:37:13,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/e8512e2c5e2e47be819e7b3b5cc24f7a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e8512e2c5e2e47be819e7b3b5cc24f7a 2024-11-07T12:37:13,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e8512e2c5e2e47be819e7b3b5cc24f7a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:37:13,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for bf2b7ddfed68e71c2777c670069c266b in 1264ms, sequenceid=155, compaction requested=true 2024-11-07T12:37:13,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:13,205 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:13,205 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:13,207 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:13,207 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:13,207 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:13,207 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/e0f332badef04ac2957e35d77fa8d298, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/af204dc360c14597b0b5bfded1a74177, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/b86e81cff75147f98b881fa32f96a86d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4167feb2adfa4b95a4dd3ddfda159bcd] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=49.7 K 2024-11-07T12:37:13,207 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:13,207 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f332badef04ac2957e35d77fa8d298, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730983028344 2024-11-07T12:37:13,207 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:13,208 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:13,208 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cbbf4b5ea2b4f6c944724761c7ee8fb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/d0a88976344949da9c72302447f6b9d2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cf518931332421cbd71564043aec810, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/325ce207fb334e4099e9e37d60bf5316] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=47.3 K 2024-11-07T12:37:13,208 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cbbf4b5ea2b4f6c944724761c7ee8fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730983028344 2024-11-07T12:37:13,208 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting af204dc360c14597b0b5bfded1a74177, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730983028994 2024-11-07T12:37:13,209 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b86e81cff75147f98b881fa32f96a86d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730983029645 2024-11-07T12:37:13,209 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a88976344949da9c72302447f6b9d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730983028994 2024-11-07T12:37:13,209 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4167feb2adfa4b95a4dd3ddfda159bcd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983031818 2024-11-07T12:37:13,210 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cf518931332421cbd71564043aec810, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730983029645 2024-11-07T12:37:13,210 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 325ce207fb334e4099e9e37d60bf5316, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983031818 2024-11-07T12:37:13,236 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:13,237 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/a624615d414b4ba7a670cb541d419a25 is 50, key is test_row_0/A:col10/1730983031819/Put/seqid=0 2024-11-07T12:37:13,242 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:13,243 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/3d33be4215304a02b4655dc08d2ab794 is 50, key is test_row_0/B:col10/1730983031819/Put/seqid=0 2024-11-07T12:37:13,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742127_1303 (size=12493) 2024-11-07T12:37:13,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742128_1304 (size=12493) 2024-11-07T12:37:13,269 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/3d33be4215304a02b4655dc08d2ab794 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3d33be4215304a02b4655dc08d2ab794 2024-11-07T12:37:13,274 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into 3d33be4215304a02b4655dc08d2ab794(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:13,274 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:13,274 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=12, startTime=1730983033205; duration=0sec 2024-11-07T12:37:13,275 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:13,275 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:13,275 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:13,277 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:13,277 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/C is initiating minor compaction (all files) 2024-11-07T12:37:13,277 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/C in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:13,277 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/115decc5bed54b09bc716a22e8c91481, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/0a318d5a6af8456e9ac1143626fd577f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/12aedffd71774e7fa7319b3960b3ac49, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e8512e2c5e2e47be819e7b3b5cc24f7a] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=47.3 K 2024-11-07T12:37:13,278 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 115decc5bed54b09bc716a22e8c91481, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730983028344 2024-11-07T12:37:13,278 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a318d5a6af8456e9ac1143626fd577f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730983028994 2024-11-07T12:37:13,278 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 12aedffd71774e7fa7319b3960b3ac49, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730983029645 2024-11-07T12:37:13,279 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e8512e2c5e2e47be819e7b3b5cc24f7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983031818 2024-11-07T12:37:13,287 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:13,287 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/5e8cfdb9e2f14a2f91f05aed77156982 is 50, key is test_row_0/C:col10/1730983031819/Put/seqid=0 2024-11-07T12:37:13,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742129_1305 (size=12493) 2024-11-07T12:37:13,664 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/a624615d414b4ba7a670cb541d419a25 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a624615d414b4ba7a670cb541d419a25 2024-11-07T12:37:13,670 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into a624615d414b4ba7a670cb541d419a25(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:13,670 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:13,670 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=12, startTime=1730983033205; duration=0sec 2024-11-07T12:37:13,670 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:13,670 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:13,708 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/5e8cfdb9e2f14a2f91f05aed77156982 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5e8cfdb9e2f14a2f91f05aed77156982 2024-11-07T12:37:13,715 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/C of bf2b7ddfed68e71c2777c670069c266b into 5e8cfdb9e2f14a2f91f05aed77156982(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:13,715 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:13,716 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/C, priority=12, startTime=1730983033205; duration=0sec 2024-11-07T12:37:13,716 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:13,716 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:37:13,777 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:13,778 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-07T12:37:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:37:13,782 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:13,783 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:13,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:13,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:13,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T12:37:13,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/ec39f434e17d43e9a00d20055f486f24 is 50, key is test_row_0/A:col10/1730983031960/Put/seqid=0 2024-11-07T12:37:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:37:13,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742130_1306 (size=12151) 2024-11-07T12:37:13,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983093916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:13,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:13,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:13,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:13,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983094028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:14,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:14,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:14,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:37:14,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983094108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:14,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983094108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:14,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983094108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:14,109 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:14,109 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:14,110 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:14,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983094117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:14,118 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983094237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:14,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:14,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:14,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/ec39f434e17d43e9a00d20055f486f24 2024-11-07T12:37:14,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/05457bf4d68f469ca21214278f76d8c8 is 50, key is test_row_0/B:col10/1730983031960/Put/seqid=0 2024-11-07T12:37:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742131_1307 (size=12151) 2024-11-07T12:37:14,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:14,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:14,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:37:14,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983094547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:14,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:14,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:14,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/05457bf4d68f469ca21214278f76d8c8 2024-11-07T12:37:14,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/06a1ba71cb5b40d8acce3fb65de9386b is 50, key is test_row_0/C:col10/1730983031960/Put/seqid=0 2024-11-07T12:37:14,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742132_1308 (size=12151) 2024-11-07T12:37:14,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:14,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:37:15,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:15,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:15,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983095057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:15,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/06a1ba71cb5b40d8acce3fb65de9386b 2024-11-07T12:37:15,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/ec39f434e17d43e9a00d20055f486f24 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/ec39f434e17d43e9a00d20055f486f24 2024-11-07T12:37:15,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/ec39f434e17d43e9a00d20055f486f24, entries=150, sequenceid=177, filesize=11.9 K 2024-11-07T12:37:15,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/05457bf4d68f469ca21214278f76d8c8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/05457bf4d68f469ca21214278f76d8c8 2024-11-07T12:37:15,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/05457bf4d68f469ca21214278f76d8c8, entries=150, sequenceid=177, filesize=11.9 K 2024-11-07T12:37:15,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/06a1ba71cb5b40d8acce3fb65de9386b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/06a1ba71cb5b40d8acce3fb65de9386b 2024-11-07T12:37:15,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/06a1ba71cb5b40d8acce3fb65de9386b, entries=150, sequenceid=177, filesize=11.9 K 2024-11-07T12:37:15,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for bf2b7ddfed68e71c2777c670069c266b in 1309ms, sequenceid=177, compaction requested=false 2024-11-07T12:37:15,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:15,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:37:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:15,173 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T12:37:15,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/54f8d4b5ea194ac9a53084f7afb96113 is 50, key is test_row_0/A:col10/1730983033903/Put/seqid=0 2024-11-07T12:37:15,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742133_1309 (size=12151) 2024-11-07T12:37:15,183 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/54f8d4b5ea194ac9a53084f7afb96113 2024-11-07T12:37:15,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/4329d9f4d51c4243bafd05c6bbc21d8f is 50, key is test_row_0/B:col10/1730983033903/Put/seqid=0 2024-11-07T12:37:15,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742134_1310 (size=12151) 2024-11-07T12:37:15,595 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/4329d9f4d51c4243bafd05c6bbc21d8f 2024-11-07T12:37:15,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/60b1bd68a7d04f0092ac836ea6f5cba2 is 50, key is test_row_0/C:col10/1730983033903/Put/seqid=0 2024-11-07T12:37:15,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742135_1311 (size=12151) 2024-11-07T12:37:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:37:16,008 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/60b1bd68a7d04f0092ac836ea6f5cba2 2024-11-07T12:37:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/54f8d4b5ea194ac9a53084f7afb96113 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/54f8d4b5ea194ac9a53084f7afb96113 2024-11-07T12:37:16,020 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/54f8d4b5ea194ac9a53084f7afb96113, entries=150, sequenceid=194, filesize=11.9 K 2024-11-07T12:37:16,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/4329d9f4d51c4243bafd05c6bbc21d8f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/4329d9f4d51c4243bafd05c6bbc21d8f 2024-11-07T12:37:16,025 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/4329d9f4d51c4243bafd05c6bbc21d8f, entries=150, sequenceid=194, filesize=11.9 K 2024-11-07T12:37:16,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/60b1bd68a7d04f0092ac836ea6f5cba2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/60b1bd68a7d04f0092ac836ea6f5cba2 2024-11-07T12:37:16,030 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/60b1bd68a7d04f0092ac836ea6f5cba2, entries=150, sequenceid=194, filesize=11.9 K 2024-11-07T12:37:16,031 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for bf2b7ddfed68e71c2777c670069c266b in 858ms, sequenceid=194, compaction requested=true 2024-11-07T12:37:16,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:16,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:16,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-07T12:37:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-07T12:37:16,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T12:37:16,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2490 sec 2024-11-07T12:37:16,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2570 sec 2024-11-07T12:37:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:16,089 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:16,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:37:16,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/a965904be99f48baa981176d82659cc8 is 50, key is test_row_0/A:col10/1730983036088/Put/seqid=0 2024-11-07T12:37:16,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742136_1312 (size=12151) 2024-11-07T12:37:16,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/a965904be99f48baa981176d82659cc8 2024-11-07T12:37:16,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/b0488adcdf784bf59c6e545220fa6152 is 50, key is test_row_0/B:col10/1730983036088/Put/seqid=0 2024-11-07T12:37:16,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742137_1313 (size=12151) 2024-11-07T12:37:16,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983096143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983096143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983096149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983096150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983096151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983096257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983096257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983096257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983096257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983096257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983096467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983096467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983096467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983096467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983096467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/b0488adcdf784bf59c6e545220fa6152 2024-11-07T12:37:16,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/13b9cd3ef93b452e8e49f1e0ec80216d is 50, key is test_row_0/C:col10/1730983036088/Put/seqid=0 2024-11-07T12:37:16,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742138_1314 (size=12151) 2024-11-07T12:37:16,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/13b9cd3ef93b452e8e49f1e0ec80216d 2024-11-07T12:37:16,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/a965904be99f48baa981176d82659cc8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a965904be99f48baa981176d82659cc8 2024-11-07T12:37:16,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a965904be99f48baa981176d82659cc8, entries=150, sequenceid=205, filesize=11.9 K 2024-11-07T12:37:16,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/b0488adcdf784bf59c6e545220fa6152 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/b0488adcdf784bf59c6e545220fa6152 2024-11-07T12:37:16,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/b0488adcdf784bf59c6e545220fa6152, entries=150, sequenceid=205, filesize=11.9 K 2024-11-07T12:37:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/13b9cd3ef93b452e8e49f1e0ec80216d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/13b9cd3ef93b452e8e49f1e0ec80216d 2024-11-07T12:37:16,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/13b9cd3ef93b452e8e49f1e0ec80216d, entries=150, sequenceid=205, filesize=11.9 K 2024-11-07T12:37:16,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for bf2b7ddfed68e71c2777c670069c266b in 465ms, sequenceid=205, compaction requested=true 2024-11-07T12:37:16,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:16,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:16,554 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:16,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:16,554 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:16,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:16,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:16,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:16,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:16,556 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:16,556 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:16,556 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:16,556 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3d33be4215304a02b4655dc08d2ab794, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/05457bf4d68f469ca21214278f76d8c8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/4329d9f4d51c4243bafd05c6bbc21d8f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/b0488adcdf784bf59c6e545220fa6152] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=47.8 K 2024-11-07T12:37:16,556 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:16,556 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:16,556 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:16,556 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a624615d414b4ba7a670cb541d419a25, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/ec39f434e17d43e9a00d20055f486f24, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/54f8d4b5ea194ac9a53084f7afb96113, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a965904be99f48baa981176d82659cc8] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=47.8 K 2024-11-07T12:37:16,557 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d33be4215304a02b4655dc08d2ab794, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983031818 2024-11-07T12:37:16,557 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a624615d414b4ba7a670cb541d419a25, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983031818 2024-11-07T12:37:16,558 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 05457bf4d68f469ca21214278f76d8c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730983031956 2024-11-07T12:37:16,558 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec39f434e17d43e9a00d20055f486f24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730983031956 2024-11-07T12:37:16,558 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4329d9f4d51c4243bafd05c6bbc21d8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1730983033903 2024-11-07T12:37:16,558 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f8d4b5ea194ac9a53084f7afb96113, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1730983033903 2024-11-07T12:37:16,559 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b0488adcdf784bf59c6e545220fa6152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1730983036068 2024-11-07T12:37:16,559 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a965904be99f48baa981176d82659cc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1730983036068 2024-11-07T12:37:16,587 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:16,588 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/50528e494bb8437cbf4c39621c27d593 is 50, key is test_row_0/B:col10/1730983036088/Put/seqid=0 2024-11-07T12:37:16,592 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:16,593 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/94957b1215c9440180f09bf8e2c72bc5 is 50, key is test_row_0/A:col10/1730983036088/Put/seqid=0 2024-11-07T12:37:16,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742139_1315 (size=12629) 2024-11-07T12:37:16,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742140_1316 (size=12629) 2024-11-07T12:37:16,615 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/94957b1215c9440180f09bf8e2c72bc5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/94957b1215c9440180f09bf8e2c72bc5 2024-11-07T12:37:16,622 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into 94957b1215c9440180f09bf8e2c72bc5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:16,622 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:16,622 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=12, startTime=1730983036554; duration=0sec 2024-11-07T12:37:16,622 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:16,622 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:16,622 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:16,624 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:16,624 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/C is initiating minor compaction (all files) 2024-11-07T12:37:16,624 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/C in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:16,624 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5e8cfdb9e2f14a2f91f05aed77156982, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/06a1ba71cb5b40d8acce3fb65de9386b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/60b1bd68a7d04f0092ac836ea6f5cba2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/13b9cd3ef93b452e8e49f1e0ec80216d] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=47.8 K 2024-11-07T12:37:16,625 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8cfdb9e2f14a2f91f05aed77156982, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983031818 2024-11-07T12:37:16,625 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a1ba71cb5b40d8acce3fb65de9386b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730983031956 2024-11-07T12:37:16,625 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60b1bd68a7d04f0092ac836ea6f5cba2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1730983033903 2024-11-07T12:37:16,626 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13b9cd3ef93b452e8e49f1e0ec80216d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1730983036068 2024-11-07T12:37:16,636 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#C#compaction#265 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:16,636 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/04eece51020c45148a9668d095d85d22 is 50, key is test_row_0/C:col10/1730983036088/Put/seqid=0 2024-11-07T12:37:16,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742141_1317 (size=12629) 2024-11-07T12:37:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:16,780 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:16,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T12:37:16,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/c9cd4e4e33914f628c8e4aa24f662059 is 50, key is test_row_0/A:col10/1730983036149/Put/seqid=0 2024-11-07T12:37:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983096782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983096782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983096782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742142_1318 (size=12151) 2024-11-07T12:37:16,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983096788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983096789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983096897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983096898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983096898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983096898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983096898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,003 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/50528e494bb8437cbf4c39621c27d593 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/50528e494bb8437cbf4c39621c27d593 2024-11-07T12:37:17,008 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into 50528e494bb8437cbf4c39621c27d593(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:17,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:17,008 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=12, startTime=1730983036554; duration=0sec 2024-11-07T12:37:17,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:17,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:17,048 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/04eece51020c45148a9668d095d85d22 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/04eece51020c45148a9668d095d85d22 2024-11-07T12:37:17,053 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/C of bf2b7ddfed68e71c2777c670069c266b into 04eece51020c45148a9668d095d85d22(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:17,053 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:17,053 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/C, priority=12, startTime=1730983036555; duration=0sec 2024-11-07T12:37:17,053 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:17,053 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:17,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983097108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983097108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983097108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983097108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983097108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/c9cd4e4e33914f628c8e4aa24f662059 2024-11-07T12:37:17,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/ca8bf49dddfe417fb7d654e018ea23ec is 50, key is test_row_0/B:col10/1730983036149/Put/seqid=0 2024-11-07T12:37:17,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742143_1319 (size=12151) 2024-11-07T12:37:17,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983097417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983097417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983097418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983097418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983097418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/ca8bf49dddfe417fb7d654e018ea23ec 2024-11-07T12:37:17,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/e3a9fb7110a24db78a1391bb45f5b9e5 is 50, key is test_row_0/C:col10/1730983036149/Put/seqid=0 2024-11-07T12:37:17,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742144_1320 (size=12151) 2024-11-07T12:37:17,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983097928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983097928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983097928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983097928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983097932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:37:17,938 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:17,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-07T12:37:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:37:17,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:17,941 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:17,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:18,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/e3a9fb7110a24db78a1391bb45f5b9e5 2024-11-07T12:37:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/c9cd4e4e33914f628c8e4aa24f662059 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c9cd4e4e33914f628c8e4aa24f662059 2024-11-07T12:37:18,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c9cd4e4e33914f628c8e4aa24f662059, entries=150, sequenceid=234, filesize=11.9 K 2024-11-07T12:37:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/ca8bf49dddfe417fb7d654e018ea23ec as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/ca8bf49dddfe417fb7d654e018ea23ec 2024-11-07T12:37:18,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/ca8bf49dddfe417fb7d654e018ea23ec, entries=150, sequenceid=234, filesize=11.9 K 2024-11-07T12:37:18,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/e3a9fb7110a24db78a1391bb45f5b9e5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e3a9fb7110a24db78a1391bb45f5b9e5 2024-11-07T12:37:18,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e3a9fb7110a24db78a1391bb45f5b9e5, entries=150, sequenceid=234, filesize=11.9 K 2024-11-07T12:37:18,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for bf2b7ddfed68e71c2777c670069c266b in 1264ms, sequenceid=234, compaction requested=false 2024-11-07T12:37:18,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:37:18,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:37:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:18,095 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T12:37:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/a52d054d19e840b8b712327b078be855 is 50, key is test_row_0/A:col10/1730983036781/Put/seqid=0 2024-11-07T12:37:18,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742145_1321 (size=12151) 2024-11-07T12:37:18,121 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/a52d054d19e840b8b712327b078be855 2024-11-07T12:37:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/e7adcc60919f48239733f6512952cee5 is 50, key is test_row_0/B:col10/1730983036781/Put/seqid=0 2024-11-07T12:37:18,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742146_1322 (size=12151) 2024-11-07T12:37:18,194 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/e7adcc60919f48239733f6512952cee5 2024-11-07T12:37:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/fa95e87b39494086bb73f6e5135d2473 is 50, key is test_row_0/C:col10/1730983036781/Put/seqid=0 2024-11-07T12:37:18,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742147_1323 (size=12151) 2024-11-07T12:37:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:37:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:37:18,614 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/fa95e87b39494086bb73f6e5135d2473 2024-11-07T12:37:18,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/a52d054d19e840b8b712327b078be855 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a52d054d19e840b8b712327b078be855 2024-11-07T12:37:18,624 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a52d054d19e840b8b712327b078be855, entries=150, sequenceid=244, filesize=11.9 K 2024-11-07T12:37:18,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/e7adcc60919f48239733f6512952cee5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e7adcc60919f48239733f6512952cee5 2024-11-07T12:37:18,630 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e7adcc60919f48239733f6512952cee5, entries=150, sequenceid=244, filesize=11.9 K 2024-11-07T12:37:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/fa95e87b39494086bb73f6e5135d2473 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa95e87b39494086bb73f6e5135d2473 2024-11-07T12:37:18,642 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa95e87b39494086bb73f6e5135d2473, entries=150, sequenceid=244, filesize=11.9 K 2024-11-07T12:37:18,643 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for bf2b7ddfed68e71c2777c670069c266b in 548ms, sequenceid=244, compaction requested=true 2024-11-07T12:37:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-07T12:37:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-07T12:37:18,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T12:37:18,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 704 msec 2024-11-07T12:37:18,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 711 msec 2024-11-07T12:37:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:18,952 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:18,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:37:18,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/2b7cabd9ed6242adb668518c66e57141 is 50, key is test_row_0/A:col10/1730983038951/Put/seqid=0 2024-11-07T12:37:18,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742148_1324 (size=14591) 2024-11-07T12:37:18,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/2b7cabd9ed6242adb668518c66e57141 2024-11-07T12:37:18,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/afc14fb4b5d6402d8a5c1ad437c70143 is 50, key is test_row_0/B:col10/1730983038951/Put/seqid=0 2024-11-07T12:37:18,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742149_1325 (size=12201) 2024-11-07T12:37:18,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/afc14fb4b5d6402d8a5c1ad437c70143 2024-11-07T12:37:18,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/372a72441ac94be8876d3dcb08bda160 is 50, key is test_row_0/C:col10/1730983038951/Put/seqid=0 2024-11-07T12:37:18,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742150_1326 (size=12201) 2024-11-07T12:37:18,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983098990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:18,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983098991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:18,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983098991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983098996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983098997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:37:19,077 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:19,078 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-07T12:37:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:37:19,079 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:19,080 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:19,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983099107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983099107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983099107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983099108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983099108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:37:19,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:37:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:19,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:19,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983099318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983099318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983099318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983099318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983099318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:37:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:19,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/372a72441ac94be8876d3dcb08bda160 2024-11-07T12:37:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:37:19,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/2b7cabd9ed6242adb668518c66e57141 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2b7cabd9ed6242adb668518c66e57141 2024-11-07T12:37:19,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2b7cabd9ed6242adb668518c66e57141, entries=200, sequenceid=257, filesize=14.2 K 2024-11-07T12:37:19,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/afc14fb4b5d6402d8a5c1ad437c70143 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/afc14fb4b5d6402d8a5c1ad437c70143 2024-11-07T12:37:19,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/afc14fb4b5d6402d8a5c1ad437c70143, entries=150, sequenceid=257, filesize=11.9 K 2024-11-07T12:37:19,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/372a72441ac94be8876d3dcb08bda160 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/372a72441ac94be8876d3dcb08bda160 2024-11-07T12:37:19,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/372a72441ac94be8876d3dcb08bda160, entries=150, sequenceid=257, filesize=11.9 K 2024-11-07T12:37:19,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for bf2b7ddfed68e71c2777c670069c266b in 462ms, sequenceid=257, compaction requested=true 2024-11-07T12:37:19,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:19,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:19,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:19,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:19,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:19,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:19,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:19,414 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:19,414 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:19,415 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51522 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:19,415 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:19,415 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:19,415 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:19,416 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,416 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,416 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/94957b1215c9440180f09bf8e2c72bc5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c9cd4e4e33914f628c8e4aa24f662059, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a52d054d19e840b8b712327b078be855, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2b7cabd9ed6242adb668518c66e57141] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=50.3 K 2024-11-07T12:37:19,416 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/50528e494bb8437cbf4c39621c27d593, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/ca8bf49dddfe417fb7d654e018ea23ec, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e7adcc60919f48239733f6512952cee5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/afc14fb4b5d6402d8a5c1ad437c70143] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=48.0 K 2024-11-07T12:37:19,416 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 50528e494bb8437cbf4c39621c27d593, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1730983036068 2024-11-07T12:37:19,416 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94957b1215c9440180f09bf8e2c72bc5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1730983036068 2024-11-07T12:37:19,417 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8bf49dddfe417fb7d654e018ea23ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730983036148 2024-11-07T12:37:19,417 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9cd4e4e33914f628c8e4aa24f662059, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730983036148 2024-11-07T12:37:19,417 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e7adcc60919f48239733f6512952cee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730983036781 2024-11-07T12:37:19,417 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a52d054d19e840b8b712327b078be855, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730983036781 2024-11-07T12:37:19,417 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting afc14fb4b5d6402d8a5c1ad437c70143, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730983038951 2024-11-07T12:37:19,417 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b7cabd9ed6242adb668518c66e57141, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730983038950 2024-11-07T12:37:19,426 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:19,426 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/3da154762315454597f796c2354517a0 is 50, key is test_row_0/B:col10/1730983038951/Put/seqid=0 2024-11-07T12:37:19,429 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:19,430 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/d9db688b4d7a40219eb7778beef40752 is 50, key is test_row_0/A:col10/1730983038951/Put/seqid=0 2024-11-07T12:37:19,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742151_1327 (size=12815) 2024-11-07T12:37:19,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742152_1328 (size=12815) 2024-11-07T12:37:19,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:37:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,539 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:37:19,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/0900f4773d984650801529575e49aecd is 50, key is test_row_0/A:col10/1730983038988/Put/seqid=0 2024-11-07T12:37:19,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742153_1329 (size=12301) 2024-11-07T12:37:19,553 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/0900f4773d984650801529575e49aecd 2024-11-07T12:37:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/87664178a0f34397aeece85ae21769e3 is 50, key is test_row_0/B:col10/1730983038988/Put/seqid=0 2024-11-07T12:37:19,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742154_1330 (size=12301) 2024-11-07T12:37:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:19,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:19,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983099636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983099637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983099637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983099639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983099641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:37:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983099747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983099747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983099748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983099757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,838 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/3da154762315454597f796c2354517a0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3da154762315454597f796c2354517a0 2024-11-07T12:37:19,839 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/d9db688b4d7a40219eb7778beef40752 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/d9db688b4d7a40219eb7778beef40752 2024-11-07T12:37:19,843 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into 3da154762315454597f796c2354517a0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:19,843 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:19,843 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=12, startTime=1730983039414; duration=0sec 2024-11-07T12:37:19,843 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:19,843 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:19,843 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:19,844 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into d9db688b4d7a40219eb7778beef40752(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:19,844 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:19,844 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=12, startTime=1730983039414; duration=0sec 2024-11-07T12:37:19,844 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:19,844 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:19,845 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:19,845 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/C is initiating minor compaction (all files) 2024-11-07T12:37:19,845 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/C in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,845 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/04eece51020c45148a9668d095d85d22, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e3a9fb7110a24db78a1391bb45f5b9e5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa95e87b39494086bb73f6e5135d2473, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/372a72441ac94be8876d3dcb08bda160] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=48.0 K 2024-11-07T12:37:19,846 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 04eece51020c45148a9668d095d85d22, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1730983036068 2024-11-07T12:37:19,846 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a9fb7110a24db78a1391bb45f5b9e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730983036148 2024-11-07T12:37:19,847 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting fa95e87b39494086bb73f6e5135d2473, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730983036781 2024-11-07T12:37:19,847 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 372a72441ac94be8876d3dcb08bda160, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730983038951 2024-11-07T12:37:19,866 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:19,867 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/49450729e2a94744b0976a57c5377f1f is 50, key is test_row_0/C:col10/1730983038951/Put/seqid=0 2024-11-07T12:37:19,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742155_1331 (size=12815) 2024-11-07T12:37:19,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983099957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983099957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983099958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,966 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/87664178a0f34397aeece85ae21769e3 2024-11-07T12:37:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983099967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:19,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/9ea339b4ebee421c99708d42edea6229 is 50, key is test_row_0/C:col10/1730983038988/Put/seqid=0 2024-11-07T12:37:19,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742156_1332 (size=12301) 2024-11-07T12:37:19,978 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/9ea339b4ebee421c99708d42edea6229 2024-11-07T12:37:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/0900f4773d984650801529575e49aecd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/0900f4773d984650801529575e49aecd 2024-11-07T12:37:19,986 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/0900f4773d984650801529575e49aecd, entries=150, sequenceid=281, filesize=12.0 K 2024-11-07T12:37:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/87664178a0f34397aeece85ae21769e3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/87664178a0f34397aeece85ae21769e3 2024-11-07T12:37:19,990 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/87664178a0f34397aeece85ae21769e3, entries=150, sequenceid=281, filesize=12.0 K 2024-11-07T12:37:19,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/9ea339b4ebee421c99708d42edea6229 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/9ea339b4ebee421c99708d42edea6229 2024-11-07T12:37:19,994 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/9ea339b4ebee421c99708d42edea6229, entries=150, sequenceid=281, filesize=12.0 K 2024-11-07T12:37:19,995 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for bf2b7ddfed68e71c2777c670069c266b in 456ms, sequenceid=281, compaction requested=false 2024-11-07T12:37:19,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:19,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:19,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-07T12:37:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-07T12:37:19,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-07T12:37:19,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 916 msec 2024-11-07T12:37:20,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 921 msec 2024-11-07T12:37:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:20,150 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:20,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:37:20,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4769a7a7b1374f3f8ee49e5973b5d5ef is 50, key is test_row_0/A:col10/1730983039637/Put/seqid=0 2024-11-07T12:37:20,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742157_1333 (size=14741) 2024-11-07T12:37:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:37:20,217 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:20,218 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-07T12:37:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:37:20,220 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:20,220 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:20,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:20,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983100266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983100267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983100267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983100267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983100278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,290 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/49450729e2a94744b0976a57c5377f1f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/49450729e2a94744b0976a57c5377f1f 2024-11-07T12:37:20,294 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/C of bf2b7ddfed68e71c2777c670069c266b into 49450729e2a94744b0976a57c5377f1f(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:20,294 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:20,294 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/C, priority=12, startTime=1730983039414; duration=0sec 2024-11-07T12:37:20,294 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:20,294 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:37:20,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:37:20,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:20,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983100378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:37:20,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:20,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:37:20,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4769a7a7b1374f3f8ee49e5973b5d5ef 2024-11-07T12:37:20,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/526cc9a4b57340efb392879f7fcc95f0 is 50, key is test_row_0/B:col10/1730983039637/Put/seqid=0 2024-11-07T12:37:20,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742158_1334 (size=12301) 2024-11-07T12:37:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983100587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:37:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983100777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983100777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983100777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983100787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:37:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:20,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:37:20,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983100897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:20,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/526cc9a4b57340efb392879f7fcc95f0 2024-11-07T12:37:20,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:37:20,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/2fbffd2f562b443887368947e98106c4 is 50, key is test_row_0/C:col10/1730983039637/Put/seqid=0 2024-11-07T12:37:20,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:20,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:20,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:20,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742159_1335 (size=12301) 2024-11-07T12:37:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:21,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:37:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:21,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:37:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:37:21,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/2fbffd2f562b443887368947e98106c4 2024-11-07T12:37:21,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4769a7a7b1374f3f8ee49e5973b5d5ef as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4769a7a7b1374f3f8ee49e5973b5d5ef 2024-11-07T12:37:21,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4769a7a7b1374f3f8ee49e5973b5d5ef, entries=200, sequenceid=296, filesize=14.4 K 2024-11-07T12:37:21,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/526cc9a4b57340efb392879f7fcc95f0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/526cc9a4b57340efb392879f7fcc95f0 2024-11-07T12:37:21,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/526cc9a4b57340efb392879f7fcc95f0, entries=150, sequenceid=296, filesize=12.0 K 2024-11-07T12:37:21,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/2fbffd2f562b443887368947e98106c4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2fbffd2f562b443887368947e98106c4 2024-11-07T12:37:21,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983101407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:21,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2fbffd2f562b443887368947e98106c4, entries=150, sequenceid=296, filesize=12.0 K 2024-11-07T12:37:21,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for bf2b7ddfed68e71c2777c670069c266b in 1263ms, sequenceid=296, compaction requested=true 2024-11-07T12:37:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:21,413 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:21,413 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:21,414 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:21,414 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:21,415 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,415 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3da154762315454597f796c2354517a0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/87664178a0f34397aeece85ae21769e3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/526cc9a4b57340efb392879f7fcc95f0] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=36.5 K 2024-11-07T12:37:21,415 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:21,415 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:21,415 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,415 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/d9db688b4d7a40219eb7778beef40752, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/0900f4773d984650801529575e49aecd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4769a7a7b1374f3f8ee49e5973b5d5ef] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=38.9 K 2024-11-07T12:37:21,415 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da154762315454597f796c2354517a0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730983038951 2024-11-07T12:37:21,416 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9db688b4d7a40219eb7778beef40752, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730983038951 2024-11-07T12:37:21,416 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 87664178a0f34397aeece85ae21769e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1730983038976 2024-11-07T12:37:21,416 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0900f4773d984650801529575e49aecd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1730983038976 2024-11-07T12:37:21,416 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 526cc9a4b57340efb392879f7fcc95f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983039636 2024-11-07T12:37:21,417 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4769a7a7b1374f3f8ee49e5973b5d5ef, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983039636 2024-11-07T12:37:21,427 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:21,427 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/77de1869d57e4f5c8c5c32bbb32e7680 is 50, key is test_row_0/A:col10/1730983039637/Put/seqid=0 2024-11-07T12:37:21,428 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:21,428 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/94d8bc44465b4d7697951cfcb2063d6b is 50, key is test_row_0/B:col10/1730983039637/Put/seqid=0 2024-11-07T12:37:21,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742160_1336 (size=13017) 2024-11-07T12:37:21,450 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/94d8bc44465b4d7697951cfcb2063d6b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/94d8bc44465b4d7697951cfcb2063d6b 2024-11-07T12:37:21,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:37:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,451 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:37:21,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742161_1337 (size=13017) 2024-11-07T12:37:21,460 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into 94d8bc44465b4d7697951cfcb2063d6b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:21,460 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:21,460 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=13, startTime=1730983041413; duration=0sec 2024-11-07T12:37:21,460 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:21,460 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:21,460 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:21,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/9b9c2ced09d14e74add48dcb1b508c4f is 50, key is test_row_0/A:col10/1730983040264/Put/seqid=0 2024-11-07T12:37:21,462 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:21,462 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/C is initiating minor compaction (all files) 2024-11-07T12:37:21,462 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/C in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,462 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/49450729e2a94744b0976a57c5377f1f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/9ea339b4ebee421c99708d42edea6229, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2fbffd2f562b443887368947e98106c4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=36.5 K 2024-11-07T12:37:21,463 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 49450729e2a94744b0976a57c5377f1f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730983038951 2024-11-07T12:37:21,463 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea339b4ebee421c99708d42edea6229, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1730983038976 2024-11-07T12:37:21,464 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/77de1869d57e4f5c8c5c32bbb32e7680 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/77de1869d57e4f5c8c5c32bbb32e7680 2024-11-07T12:37:21,465 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fbffd2f562b443887368947e98106c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983039636 2024-11-07T12:37:21,470 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into 77de1869d57e4f5c8c5c32bbb32e7680(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:21,470 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:21,470 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=13, startTime=1730983041413; duration=0sec 2024-11-07T12:37:21,470 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:21,470 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:21,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742162_1338 (size=12301) 2024-11-07T12:37:21,474 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/9b9c2ced09d14e74add48dcb1b508c4f 2024-11-07T12:37:21,475 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:21,475 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/82f001153a914fcd9c78f6e83180976a is 50, key is test_row_0/C:col10/1730983039637/Put/seqid=0 2024-11-07T12:37:21,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/35f480fe6c4f4d87927cf71b233d5bab is 50, key is test_row_0/B:col10/1730983040264/Put/seqid=0 2024-11-07T12:37:21,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742163_1339 (size=13017) 2024-11-07T12:37:21,489 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/82f001153a914fcd9c78f6e83180976a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/82f001153a914fcd9c78f6e83180976a 2024-11-07T12:37:21,493 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/C of bf2b7ddfed68e71c2777c670069c266b into 82f001153a914fcd9c78f6e83180976a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:21,493 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:21,493 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/C, priority=13, startTime=1730983041413; duration=0sec 2024-11-07T12:37:21,494 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:21,494 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742164_1340 (size=12301) 2024-11-07T12:37:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:21,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983101802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:21,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983101802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:21,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983101806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983101807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:21,810 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:21,895 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/35f480fe6c4f4d87927cf71b233d5bab 2024-11-07T12:37:21,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/2b7ff6d500bc491182a8894101bebdd8 is 50, key is test_row_0/C:col10/1730983040264/Put/seqid=0 2024-11-07T12:37:21,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742165_1341 (size=12301) 2024-11-07T12:37:21,910 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/2b7ff6d500bc491182a8894101bebdd8 2024-11-07T12:37:21,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/9b9c2ced09d14e74add48dcb1b508c4f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/9b9c2ced09d14e74add48dcb1b508c4f 2024-11-07T12:37:21,919 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/9b9c2ced09d14e74add48dcb1b508c4f, entries=150, sequenceid=320, filesize=12.0 K 2024-11-07T12:37:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983101918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:21,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/35f480fe6c4f4d87927cf71b233d5bab as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/35f480fe6c4f4d87927cf71b233d5bab 2024-11-07T12:37:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983101918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983101918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:21,924 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/35f480fe6c4f4d87927cf71b233d5bab, entries=150, sequenceid=320, filesize=12.0 K 2024-11-07T12:37:21,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/2b7ff6d500bc491182a8894101bebdd8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2b7ff6d500bc491182a8894101bebdd8 2024-11-07T12:37:21,929 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2b7ff6d500bc491182a8894101bebdd8, entries=150, sequenceid=320, filesize=12.0 K 2024-11-07T12:37:21,929 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for bf2b7ddfed68e71c2777c670069c266b in 478ms, sequenceid=320, compaction requested=false 2024-11-07T12:37:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-07T12:37:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-07T12:37:21,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T12:37:21,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7100 sec 2024-11-07T12:37:21,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7160 sec 2024-11-07T12:37:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:22,130 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:22,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:37:22,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/5751204d64184bf9a579eecc3bfac951 is 50, key is test_row_0/A:col10/1730983042130/Put/seqid=0 2024-11-07T12:37:22,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742166_1342 (size=14741) 2024-11-07T12:37:22,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/5751204d64184bf9a579eecc3bfac951 2024-11-07T12:37:22,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983102163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983102163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983102165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/708eac882cfe428e8dbfab3d45e7945d is 50, key is test_row_0/B:col10/1730983042130/Put/seqid=0 2024-11-07T12:37:22,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742167_1343 (size=12301) 2024-11-07T12:37:22,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983102277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983102277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983102277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:37:22,378 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:22,378 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-07T12:37:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:37:22,380 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:22,381 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:22,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983102417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,419 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:37:22,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983102488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983102488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983102488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T12:37:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:22,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:22,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/708eac882cfe428e8dbfab3d45e7945d 2024-11-07T12:37:22,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/cb1201001f4c4654ac7954c76aa40440 is 50, key is test_row_0/C:col10/1730983042130/Put/seqid=0 2024-11-07T12:37:22,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742168_1344 (size=12301) 2024-11-07T12:37:22,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/cb1201001f4c4654ac7954c76aa40440 2024-11-07T12:37:22,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/5751204d64184bf9a579eecc3bfac951 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/5751204d64184bf9a579eecc3bfac951 2024-11-07T12:37:22,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/5751204d64184bf9a579eecc3bfac951, entries=200, sequenceid=339, filesize=14.4 K 2024-11-07T12:37:22,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/708eac882cfe428e8dbfab3d45e7945d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/708eac882cfe428e8dbfab3d45e7945d 2024-11-07T12:37:22,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/708eac882cfe428e8dbfab3d45e7945d, entries=150, sequenceid=339, filesize=12.0 K 2024-11-07T12:37:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/cb1201001f4c4654ac7954c76aa40440 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/cb1201001f4c4654ac7954c76aa40440 2024-11-07T12:37:22,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/cb1201001f4c4654ac7954c76aa40440, entries=150, sequenceid=339, filesize=12.0 K 2024-11-07T12:37:22,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for bf2b7ddfed68e71c2777c670069c266b in 484ms, sequenceid=339, compaction requested=true 2024-11-07T12:37:22,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:22,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:22,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:22,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:22,615 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:22,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:22,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:22,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:37:22,615 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:22,616 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:22,616 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:22,616 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:22,616 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/77de1869d57e4f5c8c5c32bbb32e7680, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/9b9c2ced09d14e74add48dcb1b508c4f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/5751204d64184bf9a579eecc3bfac951] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=39.1 K 2024-11-07T12:37:22,616 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:22,616 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:22,616 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:22,616 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77de1869d57e4f5c8c5c32bbb32e7680, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983039636 2024-11-07T12:37:22,616 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/94d8bc44465b4d7697951cfcb2063d6b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/35f480fe6c4f4d87927cf71b233d5bab, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/708eac882cfe428e8dbfab3d45e7945d] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=36.7 K 2024-11-07T12:37:22,616 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9c2ced09d14e74add48dcb1b508c4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1730983040251 2024-11-07T12:37:22,617 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d8bc44465b4d7697951cfcb2063d6b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983039636 2024-11-07T12:37:22,617 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5751204d64184bf9a579eecc3bfac951, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1730983041800 2024-11-07T12:37:22,617 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f480fe6c4f4d87927cf71b233d5bab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1730983040251 2024-11-07T12:37:22,618 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 708eac882cfe428e8dbfab3d45e7945d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1730983041800 2024-11-07T12:37:22,626 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:22,626 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/aa4c12a82ba54905905835bb9a3a987f is 50, key is test_row_0/A:col10/1730983042130/Put/seqid=0 2024-11-07T12:37:22,628 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:22,629 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/49505ed1d46b4e898fd63d7e6e353dd3 is 50, key is test_row_0/B:col10/1730983042130/Put/seqid=0 2024-11-07T12:37:22,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742169_1345 (size=13119) 2024-11-07T12:37:22,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742170_1346 (size=13119) 2024-11-07T12:37:22,642 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/49505ed1d46b4e898fd63d7e6e353dd3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/49505ed1d46b4e898fd63d7e6e353dd3 2024-11-07T12:37:22,646 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into 49505ed1d46b4e898fd63d7e6e353dd3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:22,646 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:22,646 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=13, startTime=1730983042615; duration=0sec 2024-11-07T12:37:22,646 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:22,646 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:22,647 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:22,647 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:22,647 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/C is initiating minor compaction (all files) 2024-11-07T12:37:22,647 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/C in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:22,648 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/82f001153a914fcd9c78f6e83180976a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2b7ff6d500bc491182a8894101bebdd8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/cb1201001f4c4654ac7954c76aa40440] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=36.7 K 2024-11-07T12:37:22,648 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f001153a914fcd9c78f6e83180976a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983039636 2024-11-07T12:37:22,648 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b7ff6d500bc491182a8894101bebdd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1730983040251 2024-11-07T12:37:22,649 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting cb1201001f4c4654ac7954c76aa40440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1730983041800 2024-11-07T12:37:22,657 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#C#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:22,658 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/3c1aae300a72455ca757382230d297c3 is 50, key is test_row_0/C:col10/1730983042130/Put/seqid=0 2024-11-07T12:37:22,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742171_1347 (size=13119) 2024-11-07T12:37:22,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T12:37:22,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:22,688 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T12:37:22,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/cb7d1095695846b88af7a264c433c5bc is 50, key is test_row_0/A:col10/1730983042161/Put/seqid=0 2024-11-07T12:37:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:37:22,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742172_1348 (size=12301) 2024-11-07T12:37:22,703 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/cb7d1095695846b88af7a264c433c5bc 2024-11-07T12:37:22,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/27da4771f97c454eb6b26b9773d9dbb5 is 50, key is test_row_0/B:col10/1730983042161/Put/seqid=0 2024-11-07T12:37:22,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742173_1349 (size=12301) 2024-11-07T12:37:22,719 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/27da4771f97c454eb6b26b9773d9dbb5 2024-11-07T12:37:22,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/58369c29dd054be491e61ea686d6a1f0 is 50, key is test_row_0/C:col10/1730983042161/Put/seqid=0 2024-11-07T12:37:22,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742174_1350 (size=12301) 2024-11-07T12:37:22,740 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/58369c29dd054be491e61ea686d6a1f0 2024-11-07T12:37:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/cb7d1095695846b88af7a264c433c5bc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cb7d1095695846b88af7a264c433c5bc 2024-11-07T12:37:22,752 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cb7d1095695846b88af7a264c433c5bc, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T12:37:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/27da4771f97c454eb6b26b9773d9dbb5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/27da4771f97c454eb6b26b9773d9dbb5 2024-11-07T12:37:22,757 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/27da4771f97c454eb6b26b9773d9dbb5, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T12:37:22,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/58369c29dd054be491e61ea686d6a1f0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/58369c29dd054be491e61ea686d6a1f0 2024-11-07T12:37:22,762 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/58369c29dd054be491e61ea686d6a1f0, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T12:37:22,763 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for bf2b7ddfed68e71c2777c670069c266b in 75ms, sequenceid=360, compaction requested=false 2024-11-07T12:37:22,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:22,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:22,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-07T12:37:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-07T12:37:22,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T12:37:22,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 384 msec 2024-11-07T12:37:22,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 392 msec 2024-11-07T12:37:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:22,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:22,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:37:22,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/2e0e4fabfe3c4c67b83213ef6add07aa is 50, key is test_row_0/A:col10/1730983042802/Put/seqid=0 2024-11-07T12:37:22,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742175_1351 (size=12301) 2024-11-07T12:37:22,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983102845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983102846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983102850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983102957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983102958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983102958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:37:23,008 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:23,008 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-07T12:37:23,010 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:37:23,011 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:23,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:23,039 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/aa4c12a82ba54905905835bb9a3a987f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/aa4c12a82ba54905905835bb9a3a987f 2024-11-07T12:37:23,044 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into aa4c12a82ba54905905835bb9a3a987f(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:23,044 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:23,044 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=13, startTime=1730983042614; duration=0sec 2024-11-07T12:37:23,044 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:23,044 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:23,068 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/3c1aae300a72455ca757382230d297c3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3c1aae300a72455ca757382230d297c3 2024-11-07T12:37:23,072 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/C of bf2b7ddfed68e71c2777c670069c266b into 3c1aae300a72455ca757382230d297c3(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:23,072 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:23,072 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/C, priority=13, startTime=1730983042615; duration=0sec 2024-11-07T12:37:23,072 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:23,072 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:37:23,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:37:23,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983103167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983103167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983103167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/2e0e4fabfe3c4c67b83213ef6add07aa 2024-11-07T12:37:23,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/c760138bd32a4a859674d929cd5b9e16 is 50, key is test_row_0/B:col10/1730983042802/Put/seqid=0 2024-11-07T12:37:23,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742176_1352 (size=12301) 2024-11-07T12:37:23,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:37:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:37:23,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:37:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983103478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983103478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983103478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:37:23,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:23,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/c760138bd32a4a859674d929cd5b9e16 2024-11-07T12:37:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:37:23,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/944485029e7d48f7a67760597052fb59 is 50, key is test_row_0/C:col10/1730983042802/Put/seqid=0 2024-11-07T12:37:23,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742177_1353 (size=12301) 2024-11-07T12:37:23,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:37:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44924 deadline: 1730983103837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,841 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:37:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:23,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983103988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983103988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983103988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:24,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/944485029e7d48f7a67760597052fb59 2024-11-07T12:37:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/2e0e4fabfe3c4c67b83213ef6add07aa as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2e0e4fabfe3c4c67b83213ef6add07aa 2024-11-07T12:37:24,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2e0e4fabfe3c4c67b83213ef6add07aa, entries=150, sequenceid=371, filesize=12.0 K 2024-11-07T12:37:24,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/c760138bd32a4a859674d929cd5b9e16 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c760138bd32a4a859674d929cd5b9e16 2024-11-07T12:37:24,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c760138bd32a4a859674d929cd5b9e16, entries=150, sequenceid=371, filesize=12.0 K 2024-11-07T12:37:24,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/944485029e7d48f7a67760597052fb59 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/944485029e7d48f7a67760597052fb59 2024-11-07T12:37:24,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/944485029e7d48f7a67760597052fb59, entries=150, sequenceid=371, filesize=12.0 K 2024-11-07T12:37:24,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for bf2b7ddfed68e71c2777c670069c266b in 1268ms, sequenceid=371, compaction requested=true 2024-11-07T12:37:24,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:24,074 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:37:24,074 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:24,077 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:24,077 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:24,077 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:24,077 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:24,077 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:24,077 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:24,077 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/aa4c12a82ba54905905835bb9a3a987f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cb7d1095695846b88af7a264c433c5bc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2e0e4fabfe3c4c67b83213ef6add07aa] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=36.8 K 2024-11-07T12:37:24,078 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/49505ed1d46b4e898fd63d7e6e353dd3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/27da4771f97c454eb6b26b9773d9dbb5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c760138bd32a4a859674d929cd5b9e16] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=36.8 K 2024-11-07T12:37:24,078 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa4c12a82ba54905905835bb9a3a987f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1730983041800 2024-11-07T12:37:24,078 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 49505ed1d46b4e898fd63d7e6e353dd3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1730983041800 2024-11-07T12:37:24,079 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 27da4771f97c454eb6b26b9773d9dbb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983042151 2024-11-07T12:37:24,079 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb7d1095695846b88af7a264c433c5bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983042151 2024-11-07T12:37:24,079 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c760138bd32a4a859674d929cd5b9e16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1730983042802 2024-11-07T12:37:24,080 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e0e4fabfe3c4c67b83213ef6add07aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1730983042802 2024-11-07T12:37:24,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:37:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:24,089 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:37:24,092 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:24,092 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:24,093 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/e0337a6dc2934511929bc0b6e3a6e3d6 is 50, key is test_row_0/B:col10/1730983042802/Put/seqid=0 2024-11-07T12:37:24,093 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4b1284e46f55451994ba975d4bde67b0 is 50, key is test_row_0/A:col10/1730983042802/Put/seqid=0 2024-11-07T12:37:24,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/71258b1b1da743bc9e2d38266848738d is 50, key is test_row_0/A:col10/1730983042844/Put/seqid=0 2024-11-07T12:37:24,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742178_1354 (size=13221) 2024-11-07T12:37:24,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742179_1355 (size=13221) 2024-11-07T12:37:24,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742180_1356 (size=12301) 2024-11-07T12:37:24,121 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/71258b1b1da743bc9e2d38266848738d 2024-11-07T12:37:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:37:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/f4f6d4cb2c5c416ba9eef5a3275730f4 is 50, key is test_row_0/B:col10/1730983042844/Put/seqid=0 2024-11-07T12:37:24,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742181_1357 (size=12301) 2024-11-07T12:37:24,169 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/f4f6d4cb2c5c416ba9eef5a3275730f4 2024-11-07T12:37:24,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/807744dbdee449c7b5b25b5ecc09b958 is 50, key is test_row_0/C:col10/1730983042844/Put/seqid=0 2024-11-07T12:37:24,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742182_1358 (size=12301) 2024-11-07T12:37:24,209 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/807744dbdee449c7b5b25b5ecc09b958 2024-11-07T12:37:24,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/71258b1b1da743bc9e2d38266848738d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/71258b1b1da743bc9e2d38266848738d 2024-11-07T12:37:24,221 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/71258b1b1da743bc9e2d38266848738d, entries=150, sequenceid=398, filesize=12.0 K 2024-11-07T12:37:24,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/f4f6d4cb2c5c416ba9eef5a3275730f4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/f4f6d4cb2c5c416ba9eef5a3275730f4 2024-11-07T12:37:24,226 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/f4f6d4cb2c5c416ba9eef5a3275730f4, entries=150, sequenceid=398, filesize=12.0 K 2024-11-07T12:37:24,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/807744dbdee449c7b5b25b5ecc09b958 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/807744dbdee449c7b5b25b5ecc09b958 2024-11-07T12:37:24,232 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/807744dbdee449c7b5b25b5ecc09b958, entries=150, sequenceid=398, filesize=12.0 K 2024-11-07T12:37:24,233 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for bf2b7ddfed68e71c2777c670069c266b in 144ms, sequenceid=398, compaction requested=true 2024-11-07T12:37:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-07T12:37:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-07T12:37:24,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-07T12:37:24,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2260 sec 2024-11-07T12:37:24,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.2350 sec 2024-11-07T12:37:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:24,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:37:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:24,518 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/4b1284e46f55451994ba975d4bde67b0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4b1284e46f55451994ba975d4bde67b0 2024-11-07T12:37:24,521 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/e0337a6dc2934511929bc0b6e3a6e3d6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e0337a6dc2934511929bc0b6e3a6e3d6 2024-11-07T12:37:24,524 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into 4b1284e46f55451994ba975d4bde67b0(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:24,524 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:24,524 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=13, startTime=1730983044073; duration=0sec 2024-11-07T12:37:24,524 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:24,524 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:24,524 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:24,526 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:24,526 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/C is initiating minor compaction (all files) 2024-11-07T12:37:24,526 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/C in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:24,526 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3c1aae300a72455ca757382230d297c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/58369c29dd054be491e61ea686d6a1f0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/944485029e7d48f7a67760597052fb59, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/807744dbdee449c7b5b25b5ecc09b958] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=48.8 K 2024-11-07T12:37:24,527 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1aae300a72455ca757382230d297c3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1730983041800 2024-11-07T12:37:24,528 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58369c29dd054be491e61ea686d6a1f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983042151 2024-11-07T12:37:24,528 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into e0337a6dc2934511929bc0b6e3a6e3d6(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:24,528 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:24,528 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=13, startTime=1730983044074; duration=0sec 2024-11-07T12:37:24,528 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:24,528 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:24,529 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 944485029e7d48f7a67760597052fb59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1730983042802 2024-11-07T12:37:24,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/8ff47b6237bf4e1790d617a200076006 is 50, key is test_row_0/A:col10/1730983044490/Put/seqid=0 2024-11-07T12:37:24,530 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807744dbdee449c7b5b25b5ecc09b958, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1730983042842 2024-11-07T12:37:24,549 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#C#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:24,550 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/4da28dbf76c34b58b9a173d3b833e0c6 is 50, key is test_row_0/C:col10/1730983042844/Put/seqid=0 2024-11-07T12:37:24,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742183_1359 (size=14741) 2024-11-07T12:37:24,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742184_1360 (size=13255) 2024-11-07T12:37:24,602 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/4da28dbf76c34b58b9a173d3b833e0c6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/4da28dbf76c34b58b9a173d3b833e0c6 2024-11-07T12:37:24,608 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/C of bf2b7ddfed68e71c2777c670069c266b into 4da28dbf76c34b58b9a173d3b833e0c6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:24,608 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:24,608 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/C, priority=12, startTime=1730983044074; duration=0sec 2024-11-07T12:37:24,609 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:24,609 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:24,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983104730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:24,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983104837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:24,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/8ff47b6237bf4e1790d617a200076006 2024-11-07T12:37:24,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/18dc08128f00410c83cc976b376b728b is 50, key is test_row_0/B:col10/1730983044490/Put/seqid=0 2024-11-07T12:37:24,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742185_1361 (size=12301) 2024-11-07T12:37:24,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/18dc08128f00410c83cc976b376b728b 2024-11-07T12:37:25,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44864 deadline: 1730983105001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:25,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1730983105002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:25,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1730983105002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:25,006 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:25,006 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,007 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bf2b7ddfed68e71c2777c670069c266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:25,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/1986cb684c3941078947eede06ee192a is 50, key is test_row_0/C:col10/1730983044490/Put/seqid=0 2024-11-07T12:37:25,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742186_1362 (size=12301) 2024-11-07T12:37:25,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983105047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:37:25,157 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:25,158 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-07T12:37:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:37:25,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:25,172 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:25,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:37:25,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T12:37:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:25,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:25,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:25,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:25,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:25,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983105358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:25,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/1986cb684c3941078947eede06ee192a 2024-11-07T12:37:25,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/8ff47b6237bf4e1790d617a200076006 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/8ff47b6237bf4e1790d617a200076006 2024-11-07T12:37:25,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/8ff47b6237bf4e1790d617a200076006, entries=200, sequenceid=409, filesize=14.4 K 2024-11-07T12:37:25,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/18dc08128f00410c83cc976b376b728b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/18dc08128f00410c83cc976b376b728b 2024-11-07T12:37:25,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/18dc08128f00410c83cc976b376b728b, entries=150, sequenceid=409, filesize=12.0 K 2024-11-07T12:37:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/1986cb684c3941078947eede06ee192a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/1986cb684c3941078947eede06ee192a 2024-11-07T12:37:25,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/1986cb684c3941078947eede06ee192a, entries=150, sequenceid=409, filesize=12.0 K 2024-11-07T12:37:25,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for bf2b7ddfed68e71c2777c670069c266b in 932ms, sequenceid=409, compaction requested=true 2024-11-07T12:37:25,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:25,445 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:25,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:25,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:25,446 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:25,446 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:25,446 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/A is initiating minor compaction (all files) 2024-11-07T12:37:25,446 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/A in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:25,447 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4b1284e46f55451994ba975d4bde67b0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/71258b1b1da743bc9e2d38266848738d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/8ff47b6237bf4e1790d617a200076006] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=39.3 K 2024-11-07T12:37:25,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:25,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:25,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:25,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:25,449 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:25,449 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bf2b7ddfed68e71c2777c670069c266b/B is initiating minor compaction (all files) 2024-11-07T12:37:25,449 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf2b7ddfed68e71c2777c670069c266b/B in TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:25,449 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e0337a6dc2934511929bc0b6e3a6e3d6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/f4f6d4cb2c5c416ba9eef5a3275730f4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/18dc08128f00410c83cc976b376b728b] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp, totalSize=36.9 K 2024-11-07T12:37:25,449 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b1284e46f55451994ba975d4bde67b0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1730983042802 2024-11-07T12:37:25,450 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e0337a6dc2934511929bc0b6e3a6e3d6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1730983042802 2024-11-07T12:37:25,450 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71258b1b1da743bc9e2d38266848738d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1730983042842 2024-11-07T12:37:25,450 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f6d4cb2c5c416ba9eef5a3275730f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1730983042842 2024-11-07T12:37:25,451 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff47b6237bf4e1790d617a200076006, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1730983044490 2024-11-07T12:37:25,452 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 18dc08128f00410c83cc976b376b728b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1730983044490 2024-11-07T12:37:25,472 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#A#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:25,473 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/89710e7f3a654f57891cd50eeff3f8ac is 50, key is test_row_0/A:col10/1730983044490/Put/seqid=0 2024-11-07T12:37:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:37:25,479 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf2b7ddfed68e71c2777c670069c266b#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:25,479 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/98a0ae9b1ecb49468d67cc48d07faa5d is 50, key is test_row_0/B:col10/1730983044490/Put/seqid=0 2024-11-07T12:37:25,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T12:37:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:25,492 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:37:25,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742187_1363 (size=13323) 2024-11-07T12:37:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/cdb4b518a2a64deda196e9c2cd861d74 is 50, key is test_row_0/A:col10/1730983044708/Put/seqid=0 2024-11-07T12:37:25,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742188_1364 (size=13323) 2024-11-07T12:37:25,557 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/98a0ae9b1ecb49468d67cc48d07faa5d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/98a0ae9b1ecb49468d67cc48d07faa5d 2024-11-07T12:37:25,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742189_1365 (size=12301) 2024-11-07T12:37:25,568 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/B of bf2b7ddfed68e71c2777c670069c266b into 98a0ae9b1ecb49468d67cc48d07faa5d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:25,568 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:25,569 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/B, priority=13, startTime=1730983045445; duration=0sec 2024-11-07T12:37:25,569 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:25,569 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:25,569 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:37:25,570 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:37:25,570 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:37:25,570 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. because compaction request was cancelled 2024-11-07T12:37:25,570 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:37:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:25,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. as already flushing 2024-11-07T12:37:25,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983105906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:25,959 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/89710e7f3a654f57891cd50eeff3f8ac as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/89710e7f3a654f57891cd50eeff3f8ac 2024-11-07T12:37:25,969 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/cdb4b518a2a64deda196e9c2cd861d74 2024-11-07T12:37:25,973 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf2b7ddfed68e71c2777c670069c266b/A of bf2b7ddfed68e71c2777c670069c266b into 89710e7f3a654f57891cd50eeff3f8ac(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:25,973 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:25,974 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., storeName=bf2b7ddfed68e71c2777c670069c266b/A, priority=13, startTime=1730983045445; duration=0sec 2024-11-07T12:37:25,974 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:25,974 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/bd61ac881c8546e5bf81902380c9f411 is 50, key is test_row_0/B:col10/1730983044708/Put/seqid=0 2024-11-07T12:37:25,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742190_1366 (size=12301) 2024-11-07T12:37:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983106017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:26,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983106228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:37:26,337 INFO [Thread-1263 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1263. 2024-11-07T12:37:26,337 INFO [Thread-1257 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1257. 2024-11-07T12:37:26,337 DEBUG [Thread-1263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:26,337 DEBUG [Thread-1257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:26,337 DEBUG [Thread-1263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,337 DEBUG [Thread-1257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,338 DEBUG [Thread-1263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:26,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:26,338 INFO [Thread-1265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1265. 2024-11-07T12:37:26,338 DEBUG [Thread-1257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,338 DEBUG [Thread-1265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:26,338 DEBUG [Thread-1265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:26,338 DEBUG [Thread-1265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,340 INFO [Thread-1259 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1259. 2024-11-07T12:37:26,340 DEBUG [Thread-1259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:26,340 DEBUG [Thread-1259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,340 DEBUG [Thread-1259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:26,340 INFO [Thread-1261 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1261. 2024-11-07T12:37:26,340 DEBUG [Thread-1261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:26,340 DEBUG [Thread-1261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,340 DEBUG [Thread-1261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:26,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:26,394 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/bd61ac881c8546e5bf81902380c9f411 2024-11-07T12:37:26,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/5eb3f39d1c484f3e88eb9db51fa1dffb is 50, key is test_row_0/C:col10/1730983044708/Put/seqid=0 2024-11-07T12:37:26,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742191_1367 (size=12301) 2024-11-07T12:37:26,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44852 deadline: 1730983106537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:37:26,804 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/5eb3f39d1c484f3e88eb9db51fa1dffb 2024-11-07T12:37:26,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/cdb4b518a2a64deda196e9c2cd861d74 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cdb4b518a2a64deda196e9c2cd861d74 2024-11-07T12:37:26,811 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cdb4b518a2a64deda196e9c2cd861d74, entries=150, sequenceid=437, filesize=12.0 K 2024-11-07T12:37:26,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/bd61ac881c8546e5bf81902380c9f411 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/bd61ac881c8546e5bf81902380c9f411 2024-11-07T12:37:26,814 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/bd61ac881c8546e5bf81902380c9f411, entries=150, sequenceid=437, filesize=12.0 K 2024-11-07T12:37:26,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/5eb3f39d1c484f3e88eb9db51fa1dffb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5eb3f39d1c484f3e88eb9db51fa1dffb 2024-11-07T12:37:26,817 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5eb3f39d1c484f3e88eb9db51fa1dffb, entries=150, sequenceid=437, filesize=12.0 K 2024-11-07T12:37:26,818 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for bf2b7ddfed68e71c2777c670069c266b in 1327ms, sequenceid=437, compaction requested=true 2024-11-07T12:37:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-07T12:37:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-07T12:37:26,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T12:37:26,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6480 sec 2024-11-07T12:37:26,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.6640 sec 2024-11-07T12:37:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:27,029 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:27,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T12:37:27,029 INFO [Thread-1250 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1250. 2024-11-07T12:37:27,029 INFO [Thread-1254 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1254. 2024-11-07T12:37:27,029 INFO [Thread-1248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1248. 2024-11-07T12:37:27,030 DEBUG [Thread-1250 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:27,030 DEBUG [Thread-1248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:27,030 DEBUG [Thread-1254 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:27,030 DEBUG [Thread-1250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,030 DEBUG [Thread-1248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,030 DEBUG [Thread-1254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,030 DEBUG [Thread-1254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,030 DEBUG [Thread-1248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,030 DEBUG [Thread-1250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:27,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:27,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:27,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/295d17476a8b4ce6880257516bbda767 is 50, key is test_row_0/A:col10/1730983047028/Put/seqid=0 2024-11-07T12:37:27,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742192_1368 (size=9857) 2024-11-07T12:37:27,048 INFO [Thread-1252 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1252. 2024-11-07T12:37:27,048 DEBUG [Thread-1252 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:27,048 DEBUG [Thread-1252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,048 DEBUG [Thread-1252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:37:27,307 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:27,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/295d17476a8b4ce6880257516bbda767 2024-11-07T12:37:27,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/0969db063650415e97732c1ee567ccbd is 50, key is test_row_0/B:col10/1730983047028/Put/seqid=0 2024-11-07T12:37:27,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742193_1369 (size=9857) 2024-11-07T12:37:27,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/0969db063650415e97732c1ee567ccbd 2024-11-07T12:37:27,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/4799972bb74642eaaeb23c5a49db09c2 is 50, key is test_row_0/C:col10/1730983047028/Put/seqid=0 2024-11-07T12:37:27,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742194_1370 (size=9857) 2024-11-07T12:37:27,888 INFO [Thread-1246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1246. 2024-11-07T12:37:27,888 DEBUG [Thread-1246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:27,888 DEBUG [Thread-1246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,888 DEBUG [Thread-1246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3605 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3509 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3613 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3651 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3527 2024-11-07T12:37:27,889 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:37:27,889 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:37:27,889 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:37:27,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:27,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:27,890 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T12:37:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:37:27,893 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983047892"}]},"ts":"1730983047892"} 2024-11-07T12:37:27,894 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:37:27,894 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:37:27,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:37:27,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, UNASSIGN}] 2024-11-07T12:37:27,896 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, UNASSIGN 2024-11-07T12:37:27,897 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=bf2b7ddfed68e71c2777c670069c266b, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:27,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, UNASSIGN because future has completed 2024-11-07T12:37:27,898 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:37:27,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:37:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:37:28,050 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:28,050 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:37:28,050 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing bf2b7ddfed68e71c2777c670069c266b, disabling compactions & flushes 2024-11-07T12:37:28,050 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:37:28,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/4799972bb74642eaaeb23c5a49db09c2 2024-11-07T12:37:28,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/295d17476a8b4ce6880257516bbda767 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/295d17476a8b4ce6880257516bbda767 2024-11-07T12:37:28,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/295d17476a8b4ce6880257516bbda767, entries=100, sequenceid=453, filesize=9.6 K 2024-11-07T12:37:28,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/0969db063650415e97732c1ee567ccbd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0969db063650415e97732c1ee567ccbd 2024-11-07T12:37:28,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0969db063650415e97732c1ee567ccbd, entries=100, sequenceid=453, filesize=9.6 K 2024-11-07T12:37:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/4799972bb74642eaaeb23c5a49db09c2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/4799972bb74642eaaeb23c5a49db09c2 2024-11-07T12:37:28,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/4799972bb74642eaaeb23c5a49db09c2, entries=100, sequenceid=453, filesize=9.6 K 2024-11-07T12:37:28,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=13.42 KB/13740 for bf2b7ddfed68e71c2777c670069c266b in 1243ms, sequenceid=453, compaction requested=true 2024-11-07T12:37:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf2b7ddfed68e71c2777c670069c266b: 2024-11-07T12:37:28,272 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:28,272 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:28,272 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. after waiting 0 ms 2024-11-07T12:37:28,272 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. because compaction request was cancelled 2024-11-07T12:37:28,272 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:28,272 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:A 2024-11-07T12:37:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:28,272 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing bf2b7ddfed68e71c2777c670069c266b 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-07T12:37:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:B, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:28,272 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. because compaction request was cancelled 2024-11-07T12:37:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:28,272 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:B 2024-11-07T12:37:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf2b7ddfed68e71c2777c670069c266b:C, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:28,272 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. because compaction request was cancelled 2024-11-07T12:37:28,272 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf2b7ddfed68e71c2777c670069c266b:C 2024-11-07T12:37:28,276 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/fea38c7daa4c48dab3b8c51a868d386d is 50, key is test_row_0/A:col10/1730983047887/Put/seqid=0 2024-11-07T12:37:28,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742195_1371 (size=9857) 2024-11-07T12:37:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:37:28,680 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/fea38c7daa4c48dab3b8c51a868d386d 2024-11-07T12:37:28,687 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/c6448ed6fea84df6a2af4d6db547aa7d is 50, key is test_row_0/B:col10/1730983047887/Put/seqid=0 2024-11-07T12:37:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742196_1372 (size=9857) 2024-11-07T12:37:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:37:29,091 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/c6448ed6fea84df6a2af4d6db547aa7d 2024-11-07T12:37:29,097 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/3769dcc6d89c479895d6110c5ce5f8ae is 50, key is test_row_0/C:col10/1730983047887/Put/seqid=0 2024-11-07T12:37:29,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742197_1373 (size=9857) 2024-11-07T12:37:29,501 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/3769dcc6d89c479895d6110c5ce5f8ae 2024-11-07T12:37:29,505 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/A/fea38c7daa4c48dab3b8c51a868d386d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/fea38c7daa4c48dab3b8c51a868d386d 2024-11-07T12:37:29,508 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/fea38c7daa4c48dab3b8c51a868d386d, entries=100, sequenceid=458, filesize=9.6 K 2024-11-07T12:37:29,509 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/B/c6448ed6fea84df6a2af4d6db547aa7d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c6448ed6fea84df6a2af4d6db547aa7d 2024-11-07T12:37:29,512 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c6448ed6fea84df6a2af4d6db547aa7d, entries=100, sequenceid=458, filesize=9.6 K 2024-11-07T12:37:29,512 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/.tmp/C/3769dcc6d89c479895d6110c5ce5f8ae as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3769dcc6d89c479895d6110c5ce5f8ae 2024-11-07T12:37:29,515 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3769dcc6d89c479895d6110c5ce5f8ae, entries=100, sequenceid=458, filesize=9.6 K 2024-11-07T12:37:29,516 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for bf2b7ddfed68e71c2777c670069c266b in 1244ms, sequenceid=458, compaction requested=true 2024-11-07T12:37:29,516 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c1be008d93cf44d688a83bc8116aee17, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/3dc474e3eae94642838a651cf51d543e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/7c96459552f54b6a9c3bb2fe061615b4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4d6a4d6997ae43b29880652d2b69db36, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/1e55d2f229e741c0845fdde6ded60b25, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/198074e8683d4ab7921161c5fc7f4409, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/e0f332badef04ac2957e35d77fa8d298, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/af204dc360c14597b0b5bfded1a74177, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/b86e81cff75147f98b881fa32f96a86d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4167feb2adfa4b95a4dd3ddfda159bcd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a624615d414b4ba7a670cb541d419a25, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/ec39f434e17d43e9a00d20055f486f24, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/54f8d4b5ea194ac9a53084f7afb96113, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/94957b1215c9440180f09bf8e2c72bc5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a965904be99f48baa981176d82659cc8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c9cd4e4e33914f628c8e4aa24f662059, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a52d054d19e840b8b712327b078be855, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2b7cabd9ed6242adb668518c66e57141, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/d9db688b4d7a40219eb7778beef40752, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/0900f4773d984650801529575e49aecd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4769a7a7b1374f3f8ee49e5973b5d5ef, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/77de1869d57e4f5c8c5c32bbb32e7680, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/9b9c2ced09d14e74add48dcb1b508c4f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/5751204d64184bf9a579eecc3bfac951, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/aa4c12a82ba54905905835bb9a3a987f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cb7d1095695846b88af7a264c433c5bc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4b1284e46f55451994ba975d4bde67b0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2e0e4fabfe3c4c67b83213ef6add07aa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/71258b1b1da743bc9e2d38266848738d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/8ff47b6237bf4e1790d617a200076006] to archive 2024-11-07T12:37:29,517 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:29,518 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c1be008d93cf44d688a83bc8116aee17 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c1be008d93cf44d688a83bc8116aee17 2024-11-07T12:37:29,519 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/3dc474e3eae94642838a651cf51d543e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/3dc474e3eae94642838a651cf51d543e 2024-11-07T12:37:29,520 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/7c96459552f54b6a9c3bb2fe061615b4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/7c96459552f54b6a9c3bb2fe061615b4 2024-11-07T12:37:29,521 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4d6a4d6997ae43b29880652d2b69db36 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4d6a4d6997ae43b29880652d2b69db36 2024-11-07T12:37:29,522 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/1e55d2f229e741c0845fdde6ded60b25 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/1e55d2f229e741c0845fdde6ded60b25 2024-11-07T12:37:29,523 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/198074e8683d4ab7921161c5fc7f4409 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/198074e8683d4ab7921161c5fc7f4409 2024-11-07T12:37:29,524 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/e0f332badef04ac2957e35d77fa8d298 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/e0f332badef04ac2957e35d77fa8d298 2024-11-07T12:37:29,525 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/af204dc360c14597b0b5bfded1a74177 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/af204dc360c14597b0b5bfded1a74177 2024-11-07T12:37:29,526 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/b86e81cff75147f98b881fa32f96a86d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/b86e81cff75147f98b881fa32f96a86d 2024-11-07T12:37:29,527 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4167feb2adfa4b95a4dd3ddfda159bcd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4167feb2adfa4b95a4dd3ddfda159bcd 2024-11-07T12:37:29,528 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a624615d414b4ba7a670cb541d419a25 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a624615d414b4ba7a670cb541d419a25 2024-11-07T12:37:29,529 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/ec39f434e17d43e9a00d20055f486f24 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/ec39f434e17d43e9a00d20055f486f24 2024-11-07T12:37:29,530 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/54f8d4b5ea194ac9a53084f7afb96113 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/54f8d4b5ea194ac9a53084f7afb96113 2024-11-07T12:37:29,531 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/94957b1215c9440180f09bf8e2c72bc5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/94957b1215c9440180f09bf8e2c72bc5 2024-11-07T12:37:29,532 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a965904be99f48baa981176d82659cc8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a965904be99f48baa981176d82659cc8 2024-11-07T12:37:29,534 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c9cd4e4e33914f628c8e4aa24f662059 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/c9cd4e4e33914f628c8e4aa24f662059 2024-11-07T12:37:29,535 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a52d054d19e840b8b712327b078be855 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/a52d054d19e840b8b712327b078be855 2024-11-07T12:37:29,536 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2b7cabd9ed6242adb668518c66e57141 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2b7cabd9ed6242adb668518c66e57141 2024-11-07T12:37:29,537 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/d9db688b4d7a40219eb7778beef40752 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/d9db688b4d7a40219eb7778beef40752 2024-11-07T12:37:29,538 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/0900f4773d984650801529575e49aecd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/0900f4773d984650801529575e49aecd 2024-11-07T12:37:29,539 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4769a7a7b1374f3f8ee49e5973b5d5ef to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4769a7a7b1374f3f8ee49e5973b5d5ef 2024-11-07T12:37:29,540 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/77de1869d57e4f5c8c5c32bbb32e7680 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/77de1869d57e4f5c8c5c32bbb32e7680 2024-11-07T12:37:29,540 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/9b9c2ced09d14e74add48dcb1b508c4f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/9b9c2ced09d14e74add48dcb1b508c4f 2024-11-07T12:37:29,542 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/5751204d64184bf9a579eecc3bfac951 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/5751204d64184bf9a579eecc3bfac951 2024-11-07T12:37:29,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/aa4c12a82ba54905905835bb9a3a987f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/aa4c12a82ba54905905835bb9a3a987f 2024-11-07T12:37:29,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cb7d1095695846b88af7a264c433c5bc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cb7d1095695846b88af7a264c433c5bc 2024-11-07T12:37:29,544 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4b1284e46f55451994ba975d4bde67b0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/4b1284e46f55451994ba975d4bde67b0 2024-11-07T12:37:29,545 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2e0e4fabfe3c4c67b83213ef6add07aa to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/2e0e4fabfe3c4c67b83213ef6add07aa 2024-11-07T12:37:29,546 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/71258b1b1da743bc9e2d38266848738d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/71258b1b1da743bc9e2d38266848738d 2024-11-07T12:37:29,547 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/8ff47b6237bf4e1790d617a200076006 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/8ff47b6237bf4e1790d617a200076006 2024-11-07T12:37:29,549 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/941dedf33fb344688d53e2740422954f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a203151dcf6a4355aff5fe240994ac4e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a965b9733276463f8e209e1c4793ca2e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/54f5497702f64cb7879e59f165a50732, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/2026b3aca09a4dd3b9413ba9df9409da, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cbbf4b5ea2b4f6c944724761c7ee8fb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/dfe58b01896d4affa1c680dea6a77514, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/d0a88976344949da9c72302447f6b9d2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cf518931332421cbd71564043aec810, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3d33be4215304a02b4655dc08d2ab794, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/325ce207fb334e4099e9e37d60bf5316, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/05457bf4d68f469ca21214278f76d8c8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/4329d9f4d51c4243bafd05c6bbc21d8f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/50528e494bb8437cbf4c39621c27d593, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/b0488adcdf784bf59c6e545220fa6152, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/ca8bf49dddfe417fb7d654e018ea23ec, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e7adcc60919f48239733f6512952cee5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3da154762315454597f796c2354517a0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/afc14fb4b5d6402d8a5c1ad437c70143, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/87664178a0f34397aeece85ae21769e3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/94d8bc44465b4d7697951cfcb2063d6b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/526cc9a4b57340efb392879f7fcc95f0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/35f480fe6c4f4d87927cf71b233d5bab, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/49505ed1d46b4e898fd63d7e6e353dd3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/708eac882cfe428e8dbfab3d45e7945d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/27da4771f97c454eb6b26b9773d9dbb5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e0337a6dc2934511929bc0b6e3a6e3d6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c760138bd32a4a859674d929cd5b9e16, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/f4f6d4cb2c5c416ba9eef5a3275730f4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/18dc08128f00410c83cc976b376b728b] to archive 2024-11-07T12:37:29,549 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:29,551 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/941dedf33fb344688d53e2740422954f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/941dedf33fb344688d53e2740422954f 2024-11-07T12:37:29,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a203151dcf6a4355aff5fe240994ac4e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a203151dcf6a4355aff5fe240994ac4e 2024-11-07T12:37:29,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a965b9733276463f8e209e1c4793ca2e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/a965b9733276463f8e209e1c4793ca2e 2024-11-07T12:37:29,553 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/54f5497702f64cb7879e59f165a50732 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/54f5497702f64cb7879e59f165a50732 2024-11-07T12:37:29,554 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/2026b3aca09a4dd3b9413ba9df9409da to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/2026b3aca09a4dd3b9413ba9df9409da 2024-11-07T12:37:29,555 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cbbf4b5ea2b4f6c944724761c7ee8fb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cbbf4b5ea2b4f6c944724761c7ee8fb 2024-11-07T12:37:29,556 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/dfe58b01896d4affa1c680dea6a77514 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/dfe58b01896d4affa1c680dea6a77514 2024-11-07T12:37:29,557 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/d0a88976344949da9c72302447f6b9d2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/d0a88976344949da9c72302447f6b9d2 2024-11-07T12:37:29,558 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cf518931332421cbd71564043aec810 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0cf518931332421cbd71564043aec810 2024-11-07T12:37:29,559 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3d33be4215304a02b4655dc08d2ab794 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3d33be4215304a02b4655dc08d2ab794 2024-11-07T12:37:29,560 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/325ce207fb334e4099e9e37d60bf5316 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/325ce207fb334e4099e9e37d60bf5316 2024-11-07T12:37:29,561 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/05457bf4d68f469ca21214278f76d8c8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/05457bf4d68f469ca21214278f76d8c8 2024-11-07T12:37:29,562 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/4329d9f4d51c4243bafd05c6bbc21d8f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/4329d9f4d51c4243bafd05c6bbc21d8f 2024-11-07T12:37:29,564 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/50528e494bb8437cbf4c39621c27d593 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/50528e494bb8437cbf4c39621c27d593 2024-11-07T12:37:29,565 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/b0488adcdf784bf59c6e545220fa6152 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/b0488adcdf784bf59c6e545220fa6152 2024-11-07T12:37:29,566 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/ca8bf49dddfe417fb7d654e018ea23ec to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/ca8bf49dddfe417fb7d654e018ea23ec 2024-11-07T12:37:29,567 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e7adcc60919f48239733f6512952cee5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e7adcc60919f48239733f6512952cee5 2024-11-07T12:37:29,568 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3da154762315454597f796c2354517a0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/3da154762315454597f796c2354517a0 2024-11-07T12:37:29,569 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/afc14fb4b5d6402d8a5c1ad437c70143 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/afc14fb4b5d6402d8a5c1ad437c70143 2024-11-07T12:37:29,571 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/87664178a0f34397aeece85ae21769e3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/87664178a0f34397aeece85ae21769e3 2024-11-07T12:37:29,572 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/94d8bc44465b4d7697951cfcb2063d6b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/94d8bc44465b4d7697951cfcb2063d6b 2024-11-07T12:37:29,573 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/526cc9a4b57340efb392879f7fcc95f0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/526cc9a4b57340efb392879f7fcc95f0 2024-11-07T12:37:29,574 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/35f480fe6c4f4d87927cf71b233d5bab to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/35f480fe6c4f4d87927cf71b233d5bab 2024-11-07T12:37:29,575 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/49505ed1d46b4e898fd63d7e6e353dd3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/49505ed1d46b4e898fd63d7e6e353dd3 2024-11-07T12:37:29,576 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/708eac882cfe428e8dbfab3d45e7945d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/708eac882cfe428e8dbfab3d45e7945d 2024-11-07T12:37:29,577 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/27da4771f97c454eb6b26b9773d9dbb5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/27da4771f97c454eb6b26b9773d9dbb5 2024-11-07T12:37:29,578 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e0337a6dc2934511929bc0b6e3a6e3d6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/e0337a6dc2934511929bc0b6e3a6e3d6 2024-11-07T12:37:29,579 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c760138bd32a4a859674d929cd5b9e16 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c760138bd32a4a859674d929cd5b9e16 2024-11-07T12:37:29,580 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/f4f6d4cb2c5c416ba9eef5a3275730f4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/f4f6d4cb2c5c416ba9eef5a3275730f4 2024-11-07T12:37:29,581 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/18dc08128f00410c83cc976b376b728b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/18dc08128f00410c83cc976b376b728b 2024-11-07T12:37:29,582 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/41e5b5bae1394229b005ee541999d06f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/ff55279ccfac4526b45c4ab5b9a88bdc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/b173b1c5995c415c935df84c6e2a475e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/508a7128d5264d04adf0f153339db780, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/6101172039754d7dae781b4e82aa5434, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/115decc5bed54b09bc716a22e8c91481, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa4b0d6d33a14e648b7c1f29e50d15cc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/0a318d5a6af8456e9ac1143626fd577f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/12aedffd71774e7fa7319b3960b3ac49, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5e8cfdb9e2f14a2f91f05aed77156982, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e8512e2c5e2e47be819e7b3b5cc24f7a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/06a1ba71cb5b40d8acce3fb65de9386b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/60b1bd68a7d04f0092ac836ea6f5cba2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/04eece51020c45148a9668d095d85d22, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/13b9cd3ef93b452e8e49f1e0ec80216d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e3a9fb7110a24db78a1391bb45f5b9e5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa95e87b39494086bb73f6e5135d2473, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/49450729e2a94744b0976a57c5377f1f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/372a72441ac94be8876d3dcb08bda160, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/9ea339b4ebee421c99708d42edea6229, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/82f001153a914fcd9c78f6e83180976a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2fbffd2f562b443887368947e98106c4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2b7ff6d500bc491182a8894101bebdd8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3c1aae300a72455ca757382230d297c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/cb1201001f4c4654ac7954c76aa40440, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/58369c29dd054be491e61ea686d6a1f0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/944485029e7d48f7a67760597052fb59, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/807744dbdee449c7b5b25b5ecc09b958] to archive 2024-11-07T12:37:29,583 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:29,584 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/41e5b5bae1394229b005ee541999d06f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/41e5b5bae1394229b005ee541999d06f 2024-11-07T12:37:29,585 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/ff55279ccfac4526b45c4ab5b9a88bdc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/ff55279ccfac4526b45c4ab5b9a88bdc 2024-11-07T12:37:29,586 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/b173b1c5995c415c935df84c6e2a475e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/b173b1c5995c415c935df84c6e2a475e 2024-11-07T12:37:29,587 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/508a7128d5264d04adf0f153339db780 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/508a7128d5264d04adf0f153339db780 2024-11-07T12:37:29,588 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/6101172039754d7dae781b4e82aa5434 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/6101172039754d7dae781b4e82aa5434 2024-11-07T12:37:29,589 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/115decc5bed54b09bc716a22e8c91481 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/115decc5bed54b09bc716a22e8c91481 2024-11-07T12:37:29,590 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa4b0d6d33a14e648b7c1f29e50d15cc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa4b0d6d33a14e648b7c1f29e50d15cc 2024-11-07T12:37:29,591 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/0a318d5a6af8456e9ac1143626fd577f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/0a318d5a6af8456e9ac1143626fd577f 2024-11-07T12:37:29,592 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/12aedffd71774e7fa7319b3960b3ac49 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/12aedffd71774e7fa7319b3960b3ac49 2024-11-07T12:37:29,593 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5e8cfdb9e2f14a2f91f05aed77156982 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5e8cfdb9e2f14a2f91f05aed77156982 2024-11-07T12:37:29,594 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e8512e2c5e2e47be819e7b3b5cc24f7a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e8512e2c5e2e47be819e7b3b5cc24f7a 2024-11-07T12:37:29,595 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/06a1ba71cb5b40d8acce3fb65de9386b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/06a1ba71cb5b40d8acce3fb65de9386b 2024-11-07T12:37:29,596 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/60b1bd68a7d04f0092ac836ea6f5cba2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/60b1bd68a7d04f0092ac836ea6f5cba2 2024-11-07T12:37:29,597 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/04eece51020c45148a9668d095d85d22 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/04eece51020c45148a9668d095d85d22 2024-11-07T12:37:29,598 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/13b9cd3ef93b452e8e49f1e0ec80216d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/13b9cd3ef93b452e8e49f1e0ec80216d 2024-11-07T12:37:29,599 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e3a9fb7110a24db78a1391bb45f5b9e5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/e3a9fb7110a24db78a1391bb45f5b9e5 2024-11-07T12:37:29,600 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa95e87b39494086bb73f6e5135d2473 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/fa95e87b39494086bb73f6e5135d2473 2024-11-07T12:37:29,601 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/49450729e2a94744b0976a57c5377f1f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/49450729e2a94744b0976a57c5377f1f 2024-11-07T12:37:29,602 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/372a72441ac94be8876d3dcb08bda160 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/372a72441ac94be8876d3dcb08bda160 2024-11-07T12:37:29,603 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/9ea339b4ebee421c99708d42edea6229 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/9ea339b4ebee421c99708d42edea6229 2024-11-07T12:37:29,603 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/82f001153a914fcd9c78f6e83180976a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/82f001153a914fcd9c78f6e83180976a 2024-11-07T12:37:29,604 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2fbffd2f562b443887368947e98106c4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2fbffd2f562b443887368947e98106c4 2024-11-07T12:37:29,605 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2b7ff6d500bc491182a8894101bebdd8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/2b7ff6d500bc491182a8894101bebdd8 2024-11-07T12:37:29,606 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3c1aae300a72455ca757382230d297c3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3c1aae300a72455ca757382230d297c3 2024-11-07T12:37:29,607 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/cb1201001f4c4654ac7954c76aa40440 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/cb1201001f4c4654ac7954c76aa40440 2024-11-07T12:37:29,607 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/58369c29dd054be491e61ea686d6a1f0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/58369c29dd054be491e61ea686d6a1f0 2024-11-07T12:37:29,608 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/944485029e7d48f7a67760597052fb59 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/944485029e7d48f7a67760597052fb59 2024-11-07T12:37:29,609 DEBUG [StoreCloser-TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/807744dbdee449c7b5b25b5ecc09b958 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/807744dbdee449c7b5b25b5ecc09b958 2024-11-07T12:37:29,613 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/recovered.edits/461.seqid, newMaxSeqId=461, maxSeqId=1 2024-11-07T12:37:29,613 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. 2024-11-07T12:37:29,613 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for bf2b7ddfed68e71c2777c670069c266b: Waiting for close lock at 1730983048050Running coprocessor pre-close hooks at 1730983048050Disabling compacts and flushes for region at 1730983048050Disabling writes for close at 1730983048272 (+222 ms)Obtaining lock to block concurrent updates at 1730983048272Preparing flush snapshotting stores in bf2b7ddfed68e71c2777c670069c266b at 1730983048272Finished memstore snapshotting TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1730983048273 (+1 ms)Flushing stores of TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b. at 1730983048273Flushing bf2b7ddfed68e71c2777c670069c266b/A: creating writer at 1730983048273Flushing bf2b7ddfed68e71c2777c670069c266b/A: appending metadata at 1730983048276 (+3 ms)Flushing bf2b7ddfed68e71c2777c670069c266b/A: closing flushed file at 1730983048276Flushing bf2b7ddfed68e71c2777c670069c266b/B: creating writer at 1730983048684 (+408 ms)Flushing bf2b7ddfed68e71c2777c670069c266b/B: appending metadata at 1730983048686 (+2 ms)Flushing bf2b7ddfed68e71c2777c670069c266b/B: closing flushed file at 1730983048686Flushing bf2b7ddfed68e71c2777c670069c266b/C: creating writer at 1730983049095 (+409 ms)Flushing bf2b7ddfed68e71c2777c670069c266b/C: appending metadata at 1730983049097 (+2 ms)Flushing bf2b7ddfed68e71c2777c670069c266b/C: closing flushed file at 1730983049097Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21339215: reopening flushed file at 1730983049504 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47e1b2b8: reopening flushed file at 1730983049508 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@718e68af: reopening flushed file at 1730983049512 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for bf2b7ddfed68e71c2777c670069c266b in 1244ms, sequenceid=458, compaction requested=true at 1730983049516 (+4 ms)Writing region close event to WAL at 1730983049610 (+94 ms)Running coprocessor post-close hooks at 1730983049613 (+3 ms)Closed at 1730983049613 2024-11-07T12:37:29,615 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:29,616 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=bf2b7ddfed68e71c2777c670069c266b, regionState=CLOSED 2024-11-07T12:37:29,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:37:29,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-07T12:37:29,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure bf2b7ddfed68e71c2777c670069c266b, server=87fb68174c52,38515,1730982961782 in 1.7190 sec 2024-11-07T12:37:29,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-07T12:37:29,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf2b7ddfed68e71c2777c670069c266b, UNASSIGN in 1.7240 sec 2024-11-07T12:37:29,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T12:37:29,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7270 sec 2024-11-07T12:37:29,624 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983049624"}]},"ts":"1730983049624"} 2024-11-07T12:37:29,625 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:37:29,625 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:37:29,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7360 sec 2024-11-07T12:37:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:37:30,038 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:30,038 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T12:37:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:30,040 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T12:37:30,041 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:30,043 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:30,045 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/recovered.edits] 2024-11-07T12:37:30,047 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/295d17476a8b4ce6880257516bbda767 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/295d17476a8b4ce6880257516bbda767 2024-11-07T12:37:30,048 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/89710e7f3a654f57891cd50eeff3f8ac to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/89710e7f3a654f57891cd50eeff3f8ac 2024-11-07T12:37:30,049 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cdb4b518a2a64deda196e9c2cd861d74 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/cdb4b518a2a64deda196e9c2cd861d74 2024-11-07T12:37:30,050 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/fea38c7daa4c48dab3b8c51a868d386d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/A/fea38c7daa4c48dab3b8c51a868d386d 2024-11-07T12:37:30,052 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0969db063650415e97732c1ee567ccbd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/0969db063650415e97732c1ee567ccbd 2024-11-07T12:37:30,053 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/98a0ae9b1ecb49468d67cc48d07faa5d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/98a0ae9b1ecb49468d67cc48d07faa5d 2024-11-07T12:37:30,054 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/bd61ac881c8546e5bf81902380c9f411 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/bd61ac881c8546e5bf81902380c9f411 2024-11-07T12:37:30,055 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c6448ed6fea84df6a2af4d6db547aa7d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/B/c6448ed6fea84df6a2af4d6db547aa7d 2024-11-07T12:37:30,057 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/1986cb684c3941078947eede06ee192a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/1986cb684c3941078947eede06ee192a 2024-11-07T12:37:30,058 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3769dcc6d89c479895d6110c5ce5f8ae to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/3769dcc6d89c479895d6110c5ce5f8ae 2024-11-07T12:37:30,059 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/4799972bb74642eaaeb23c5a49db09c2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/4799972bb74642eaaeb23c5a49db09c2 2024-11-07T12:37:30,060 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/4da28dbf76c34b58b9a173d3b833e0c6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/4da28dbf76c34b58b9a173d3b833e0c6 2024-11-07T12:37:30,062 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5eb3f39d1c484f3e88eb9db51fa1dffb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/C/5eb3f39d1c484f3e88eb9db51fa1dffb 2024-11-07T12:37:30,064 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/recovered.edits/461.seqid to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b/recovered.edits/461.seqid 2024-11-07T12:37:30,065 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bf2b7ddfed68e71c2777c670069c266b 2024-11-07T12:37:30,065 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:37:30,067 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:30,071 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:37:30,072 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:37:30,073 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:30,073 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:37:30,074 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983050073"}]},"ts":"9223372036854775807"} 2024-11-07T12:37:30,076 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:37:30,076 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bf2b7ddfed68e71c2777c670069c266b, NAME => 'TestAcidGuarantees,,1730983024136.bf2b7ddfed68e71c2777c670069c266b.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:37:30,076 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:37:30,076 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983050076"}]},"ts":"9223372036854775807"} 2024-11-07T12:37:30,077 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:37:30,078 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:30,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-07T12:37:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T12:37:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:37:30,148 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:30,158 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242 (was 242), OpenFileDescriptor=461 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=411 (was 475), ProcessCount=11 (was 11), AvailableMemoryMB=6981 (was 7270) 2024-11-07T12:37:30,167 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=411, ProcessCount=11, AvailableMemoryMB=6980 2024-11-07T12:37:30,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:37:30,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:37:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:30,170 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:37:30,171 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:30,171 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-07T12:37:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T12:37:30,171 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:37:30,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742198_1374 (size=959) 2024-11-07T12:37:30,246 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:37:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T12:37:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T12:37:30,578 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e 2024-11-07T12:37:30,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742199_1375 (size=53) 2024-11-07T12:37:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T12:37:30,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:37:30,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2e9c3a5f1845c325e45e01815b97b02f, disabling compactions & flushes 2024-11-07T12:37:30,985 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:30,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:30,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. after waiting 0 ms 2024-11-07T12:37:30,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:30,985 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:30,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2e9c3a5f1845c325e45e01815b97b02f: Waiting for close lock at 1730983050985Disabling compacts and flushes for region at 1730983050985Disabling writes for close at 1730983050985Writing region close event to WAL at 1730983050985Closed at 1730983050985 2024-11-07T12:37:30,987 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:37:30,987 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983050987"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983050987"}]},"ts":"1730983050987"} 2024-11-07T12:37:30,988 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:37:30,989 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:37:30,989 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983050989"}]},"ts":"1730983050989"} 2024-11-07T12:37:30,990 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:37:30,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, ASSIGN}] 2024-11-07T12:37:30,992 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, ASSIGN 2024-11-07T12:37:30,992 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, ASSIGN; state=OFFLINE, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=false 2024-11-07T12:37:31,143 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=2e9c3a5f1845c325e45e01815b97b02f, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:31,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, ASSIGN because future has completed 2024-11-07T12:37:31,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:37:31,299 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:31,299 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:37:31,300 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,300 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:37:31,300 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,300 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,301 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,302 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9c3a5f1845c325e45e01815b97b02f columnFamilyName A 2024-11-07T12:37:31,302 DEBUG [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:31,303 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(327): Store=2e9c3a5f1845c325e45e01815b97b02f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:31,303 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,304 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9c3a5f1845c325e45e01815b97b02f columnFamilyName B 2024-11-07T12:37:31,304 DEBUG [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:31,304 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(327): Store=2e9c3a5f1845c325e45e01815b97b02f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:31,304 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,305 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9c3a5f1845c325e45e01815b97b02f columnFamilyName C 2024-11-07T12:37:31,305 DEBUG [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:31,306 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(327): Store=2e9c3a5f1845c325e45e01815b97b02f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:31,306 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,306 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,307 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T12:37:31,308 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,308 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,308 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:37:31,309 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,311 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:37:31,311 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 2e9c3a5f1845c325e45e01815b97b02f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66642850, jitterRate=-0.006944149732589722}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:37:31,311 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:31,312 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 2e9c3a5f1845c325e45e01815b97b02f: Running coprocessor pre-open hook at 1730983051300Writing region info on filesystem at 1730983051300Initializing all the Stores at 1730983051301 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983051301Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983051301Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983051301Cleaning up temporary data from old regions at 1730983051308 (+7 ms)Running coprocessor post-open hooks at 1730983051311 (+3 ms)Region opened successfully at 1730983051312 (+1 ms) 2024-11-07T12:37:31,312 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., pid=91, masterSystemTime=1730983051296 2024-11-07T12:37:31,314 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:31,314 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:31,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=2e9c3a5f1845c325e45e01815b97b02f, regionState=OPEN, openSeqNum=2, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:31,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:37:31,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T12:37:31,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 in 173 msec 2024-11-07T12:37:31,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-07T12:37:31,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, ASSIGN in 327 msec 2024-11-07T12:37:31,320 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:37:31,321 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983051321"}]},"ts":"1730983051321"} 2024-11-07T12:37:31,322 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:37:31,322 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:37:31,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-07T12:37:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T12:37:32,318 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:32,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:32,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:32,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:32,320 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2132fb46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:32,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f375cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:32,322 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:32,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:32,325 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:37:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb68174c52,36251,1730982961115 2024-11-07T12:37:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59646fb0 2024-11-07T12:37:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:37:32,326 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:37:32,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:37:32,327 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:37:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:32,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742200_1376 (size=995) 2024-11-07T12:37:32,738 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T12:37:32,738 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T12:37:32,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:37:32,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, REOPEN/MOVE}] 2024-11-07T12:37:32,743 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, REOPEN/MOVE 2024-11-07T12:37:32,743 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=2e9c3a5f1845c325e45e01815b97b02f, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:32,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, REOPEN/MOVE because future has completed 2024-11-07T12:37:32,745 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:37:32,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:37:32,897 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:32,897 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:37:32,897 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 2e9c3a5f1845c325e45e01815b97b02f, disabling compactions & flushes 2024-11-07T12:37:32,897 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:32,897 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:32,897 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. after waiting 0 ms 2024-11-07T12:37:32,897 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:32,901 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:37:32,902 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:32,902 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 2e9c3a5f1845c325e45e01815b97b02f: Waiting for close lock at 1730983052897Running coprocessor pre-close hooks at 1730983052897Disabling compacts and flushes for region at 1730983052897Disabling writes for close at 1730983052897Writing region close event to WAL at 1730983052898 (+1 ms)Running coprocessor post-close hooks at 1730983052902 (+4 ms)Closed at 1730983052902 2024-11-07T12:37:32,902 WARN [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 2e9c3a5f1845c325e45e01815b97b02f to self. 2024-11-07T12:37:32,904 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:32,904 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=2e9c3a5f1845c325e45e01815b97b02f, regionState=CLOSED 2024-11-07T12:37:32,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:37:32,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-07T12:37:32,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 in 161 msec 2024-11-07T12:37:32,908 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, REOPEN/MOVE; state=CLOSED, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=true 2024-11-07T12:37:33,061 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=2e9c3a5f1845c325e45e01815b97b02f, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, REOPEN/MOVE because future has completed 2024-11-07T12:37:33,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:37:33,218 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,218 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:37:33,218 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,218 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:37:33,218 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,218 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,219 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,220 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9c3a5f1845c325e45e01815b97b02f columnFamilyName A 2024-11-07T12:37:33,221 DEBUG [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:33,221 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(327): Store=2e9c3a5f1845c325e45e01815b97b02f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:33,222 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,222 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9c3a5f1845c325e45e01815b97b02f columnFamilyName B 2024-11-07T12:37:33,222 DEBUG [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:33,222 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(327): Store=2e9c3a5f1845c325e45e01815b97b02f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:33,222 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,223 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e9c3a5f1845c325e45e01815b97b02f columnFamilyName C 2024-11-07T12:37:33,223 DEBUG [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:33,223 INFO [StoreOpener-2e9c3a5f1845c325e45e01815b97b02f-1 {}] regionserver.HStore(327): Store=2e9c3a5f1845c325e45e01815b97b02f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:37:33,223 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,224 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,225 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,225 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,225 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,226 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:37:33,227 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,228 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 2e9c3a5f1845c325e45e01815b97b02f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66432203, jitterRate=-0.010083034634590149}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:37:33,228 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,229 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 2e9c3a5f1845c325e45e01815b97b02f: Running coprocessor pre-open hook at 1730983053218Writing region info on filesystem at 1730983053218Initializing all the Stores at 1730983053219 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983053219Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983053219Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983053219Cleaning up temporary data from old regions at 1730983053225 (+6 ms)Running coprocessor post-open hooks at 1730983053228 (+3 ms)Region opened successfully at 1730983053229 (+1 ms) 2024-11-07T12:37:33,232 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., pid=96, masterSystemTime=1730983053215 2024-11-07T12:37:33,234 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,234 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,235 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=2e9c3a5f1845c325e45e01815b97b02f, regionState=OPEN, openSeqNum=5, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:37:33,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-07T12:37:33,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 in 176 msec 2024-11-07T12:37:33,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-07T12:37:33,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, REOPEN/MOVE in 498 msec 2024-11-07T12:37:33,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-07T12:37:33,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-11-07T12:37:33,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-07T12:37:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-07T12:37:33,248 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:33,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,250 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbeded7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302a891e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,253 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d50f7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,254 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb2bda2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,256 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@646fda1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,257 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19763462, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,259 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7d098b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,260 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4beadef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,262 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5022f316, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,263 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df0cec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,264 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7a6de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,265 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3030616d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,267 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8b515c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@281859f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,270 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1405dc37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,271 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44aa19cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,272 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c97043a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,273 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1144c47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:37:33,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:37:33,275 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b386ec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:33,276 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:37:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63259e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:37:33,277 DEBUG [Thread-1691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,277 DEBUG [Thread-1697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,277 DEBUG [Thread-1702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,277 DEBUG [Thread-1708 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,277 DEBUG [Thread-1700 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,277 DEBUG [Thread-1704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,278 DEBUG [Thread-1689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,278 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,279 DEBUG [Thread-1693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-07T12:37:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,280 DEBUG [Thread-1695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,281 DEBUG [Thread-1706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:37:33,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:37:33,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,281 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:33,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:37:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:37:33,282 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:33,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:33,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:37:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:37:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:33,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:37:33,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cff81b882eb0479da1f97e483ad5e596_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983053285/Put/seqid=0 2024-11-07T12:37:33,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742201_1377 (size=17034) 2024-11-07T12:37:33,335 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:33,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cff81b882eb0479da1f97e483ad5e596_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cff81b882eb0479da1f97e483ad5e596_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:33,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/9294299a30804d768b204db9caa38dc8, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:33,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/9294299a30804d768b204db9caa38dc8 is 175, key is test_row_0/A:col10/1730983053285/Put/seqid=0 2024-11-07T12:37:33,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742202_1378 (size=48139) 2024-11-07T12:37:33,351 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/9294299a30804d768b204db9caa38dc8 2024-11-07T12:37:33,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983113349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983113353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983113356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983113358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983113359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/c9c2fa3e9c574e2eac1b37d07da625bf is 50, key is test_row_0/B:col10/1730983053285/Put/seqid=0 2024-11-07T12:37:33,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742203_1379 (size=12001) 2024-11-07T12:37:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:37:33,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:37:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:33,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983113468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983113468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983113477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983113477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983113477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:37:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:37:33,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983113687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983113687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983113687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983113697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983113697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:33,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:37:33,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:33,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/c9c2fa3e9c574e2eac1b37d07da625bf 2024-11-07T12:37:33,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/95501280adff47508edfb1a40c3aa5cf is 50, key is test_row_0/C:col10/1730983053285/Put/seqid=0 2024-11-07T12:37:33,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742204_1380 (size=12001) 2024-11-07T12:37:33,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:37:33,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:33,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:33,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:37:34,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983113997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983113998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983113998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983114008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983114008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:37:34,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:34,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:34,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:34,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:37:34,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:34,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:34,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:34,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:34,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:34,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/95501280adff47508edfb1a40c3aa5cf 2024-11-07T12:37:34,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/9294299a30804d768b204db9caa38dc8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9294299a30804d768b204db9caa38dc8 2024-11-07T12:37:34,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9294299a30804d768b204db9caa38dc8, entries=250, sequenceid=16, filesize=47.0 K 2024-11-07T12:37:34,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/c9c2fa3e9c574e2eac1b37d07da625bf as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/c9c2fa3e9c574e2eac1b37d07da625bf 2024-11-07T12:37:34,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/c9c2fa3e9c574e2eac1b37d07da625bf, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T12:37:34,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/95501280adff47508edfb1a40c3aa5cf as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/95501280adff47508edfb1a40c3aa5cf 2024-11-07T12:37:34,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/95501280adff47508edfb1a40c3aa5cf, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T12:37:34,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2e9c3a5f1845c325e45e01815b97b02f in 939ms, sequenceid=16, compaction requested=false 2024-11-07T12:37:34,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:34,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:37:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:34,360 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:37:34,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110728e60b982026449fad35ef152b126c5a_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983053358/Put/seqid=0 2024-11-07T12:37:34,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742205_1381 (size=12154) 2024-11-07T12:37:34,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:34,394 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110728e60b982026449fad35ef152b126c5a_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110728e60b982026449fad35ef152b126c5a_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:34,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/24e5933542cc4c44a210cfce75310658, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/24e5933542cc4c44a210cfce75310658 is 175, key is test_row_0/A:col10/1730983053358/Put/seqid=0 2024-11-07T12:37:34,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742206_1382 (size=30955) 2024-11-07T12:37:34,405 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/24e5933542cc4c44a210cfce75310658 2024-11-07T12:37:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/819c82eecc1d44cb870fe0f30e398eff is 50, key is test_row_0/B:col10/1730983053358/Put/seqid=0 2024-11-07T12:37:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:37:34,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742207_1383 (size=12001) 2024-11-07T12:37:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:34,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:34,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983114527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983114527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983114534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983114535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983114537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983114648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983114652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983114657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,830 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/819c82eecc1d44cb870fe0f30e398eff 2024-11-07T12:37:34,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/655073c81c0548019951a2862cde9621 is 50, key is test_row_0/C:col10/1730983053358/Put/seqid=0 2024-11-07T12:37:34,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742208_1384 (size=12001) 2024-11-07T12:37:34,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983114857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983114857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:34,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983114878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,141 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:37:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983115181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983115181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983115187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,246 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/655073c81c0548019951a2862cde9621 2024-11-07T12:37:35,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/24e5933542cc4c44a210cfce75310658 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/24e5933542cc4c44a210cfce75310658 2024-11-07T12:37:35,257 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/24e5933542cc4c44a210cfce75310658, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T12:37:35,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/819c82eecc1d44cb870fe0f30e398eff as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/819c82eecc1d44cb870fe0f30e398eff 2024-11-07T12:37:35,261 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/819c82eecc1d44cb870fe0f30e398eff, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T12:37:35,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/655073c81c0548019951a2862cde9621 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/655073c81c0548019951a2862cde9621 2024-11-07T12:37:35,269 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/655073c81c0548019951a2862cde9621, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T12:37:35,270 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2e9c3a5f1845c325e45e01815b97b02f in 910ms, sequenceid=40, compaction requested=false 2024-11-07T12:37:35,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:35,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:35,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-07T12:37:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-07T12:37:35,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-07T12:37:35,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9890 sec 2024-11-07T12:37:35,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.9960 sec 2024-11-07T12:37:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:37:35,428 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:35,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-07T12:37:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:37:35,431 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:35,431 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:35,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:37:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:35,553 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:35,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:37:35,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110728e448d790d9416eb3db3e97dd05959c_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983054513/Put/seqid=0 2024-11-07T12:37:35,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742209_1385 (size=14594) 2024-11-07T12:37:35,583 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:35,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:37:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:35,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,589 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110728e448d790d9416eb3db3e97dd05959c_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110728e448d790d9416eb3db3e97dd05959c_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:35,590 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/36bea73ed7bd44f5830e3b307e16dc8c, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:35,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/36bea73ed7bd44f5830e3b307e16dc8c is 175, key is test_row_0/A:col10/1730983054513/Put/seqid=0 2024-11-07T12:37:35,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742210_1386 (size=39549) 2024-11-07T12:37:35,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983115685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983115688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983115708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983115708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983115708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:37:35,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:35,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:35,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:35,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:37:35,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983115808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983115808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:35,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:35,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:37:35,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:35,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:35,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:35,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:35,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/36bea73ed7bd44f5830e3b307e16dc8c 2024-11-07T12:37:36,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/75ca896df6c840ca9061b86dac8d9b8e is 50, key is test_row_0/B:col10/1730983054513/Put/seqid=0 2024-11-07T12:37:36,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983116029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983116030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742211_1387 (size=12001) 2024-11-07T12:37:36,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:37:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/75ca896df6c840ca9061b86dac8d9b8e 2024-11-07T12:37:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:37:36,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/04d9a4f901f349a6b9689bfc91d32718 is 50, key is test_row_0/C:col10/1730983054513/Put/seqid=0 2024-11-07T12:37:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742212_1388 (size=12001) 2024-11-07T12:37:36,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:37:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983116347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:37:36,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:36,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983116348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:37:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:36,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/04d9a4f901f349a6b9689bfc91d32718 2024-11-07T12:37:36,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/36bea73ed7bd44f5830e3b307e16dc8c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36bea73ed7bd44f5830e3b307e16dc8c 2024-11-07T12:37:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:37:36,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36bea73ed7bd44f5830e3b307e16dc8c, entries=200, sequenceid=54, filesize=38.6 K 2024-11-07T12:37:36,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/75ca896df6c840ca9061b86dac8d9b8e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/75ca896df6c840ca9061b86dac8d9b8e 2024-11-07T12:37:36,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/75ca896df6c840ca9061b86dac8d9b8e, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T12:37:36,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/04d9a4f901f349a6b9689bfc91d32718 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/04d9a4f901f349a6b9689bfc91d32718 2024-11-07T12:37:36,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/04d9a4f901f349a6b9689bfc91d32718, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T12:37:36,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2e9c3a5f1845c325e45e01815b97b02f in 1028ms, sequenceid=54, compaction requested=true 2024-11-07T12:37:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:36,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:36,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:36,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:36,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:36,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:36,582 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:36,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:37:36,582 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:36,583 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:36,583 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:36,583 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/A is initiating minor compaction (all files) 2024-11-07T12:37:36,583 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/B is initiating minor compaction (all files) 2024-11-07T12:37:36,583 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/B in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,583 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/A in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,583 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9294299a30804d768b204db9caa38dc8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/24e5933542cc4c44a210cfce75310658, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36bea73ed7bd44f5830e3b307e16dc8c] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=115.9 K 2024-11-07T12:37:36,583 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/c9c2fa3e9c574e2eac1b37d07da625bf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/819c82eecc1d44cb870fe0f30e398eff, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/75ca896df6c840ca9061b86dac8d9b8e] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=35.2 K 2024-11-07T12:37:36,583 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,583 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9294299a30804d768b204db9caa38dc8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/24e5933542cc4c44a210cfce75310658, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36bea73ed7bd44f5830e3b307e16dc8c] 2024-11-07T12:37:36,584 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9294299a30804d768b204db9caa38dc8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730983053285 2024-11-07T12:37:36,584 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c9c2fa3e9c574e2eac1b37d07da625bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730983053285 2024-11-07T12:37:36,584 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e5933542cc4c44a210cfce75310658, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730983053346 2024-11-07T12:37:36,584 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 819c82eecc1d44cb870fe0f30e398eff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730983053346 2024-11-07T12:37:36,585 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36bea73ed7bd44f5830e3b307e16dc8c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730983054513 2024-11-07T12:37:36,585 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ca896df6c840ca9061b86dac8d9b8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730983054513 2024-11-07T12:37:36,610 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:36,613 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#B#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:36,613 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/bf86d0f0e20644908e85aa2324fa1893 is 50, key is test_row_0/B:col10/1730983054513/Put/seqid=0 2024-11-07T12:37:36,624 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411073b554574382544638a4483829ffb9ec1_2e9c3a5f1845c325e45e01815b97b02f store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:36,626 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411073b554574382544638a4483829ffb9ec1_2e9c3a5f1845c325e45e01815b97b02f, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:36,626 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b554574382544638a4483829ffb9ec1_2e9c3a5f1845c325e45e01815b97b02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:36,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742213_1389 (size=12104) 2024-11-07T12:37:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742214_1390 (size=4469) 2024-11-07T12:37:36,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:37:36,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:36,678 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:37:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fdf85afcd0ae494583178d0d8ae2cdc3_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983055683/Put/seqid=0 2024-11-07T12:37:36,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742215_1391 (size=12154) 2024-11-07T12:37:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:36,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:36,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983116786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983116786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983116803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983116867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983116868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983116907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983116907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:36,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983116932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,049 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#A#compaction#331 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:37,050 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/1adc08a8e13b4a12a3cb95f3773595dc is 175, key is test_row_0/A:col10/1730983054513/Put/seqid=0 2024-11-07T12:37:37,056 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/bf86d0f0e20644908e85aa2324fa1893 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/bf86d0f0e20644908e85aa2324fa1893 2024-11-07T12:37:37,060 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/B of 2e9c3a5f1845c325e45e01815b97b02f into bf86d0f0e20644908e85aa2324fa1893(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:37,060 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:37,060 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/B, priority=13, startTime=1730983056582; duration=0sec 2024-11-07T12:37:37,060 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:37,061 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:B 2024-11-07T12:37:37,061 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:37,062 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:37,062 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/C is initiating minor compaction (all files) 2024-11-07T12:37:37,062 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/C in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:37,062 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/95501280adff47508edfb1a40c3aa5cf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/655073c81c0548019951a2862cde9621, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/04d9a4f901f349a6b9689bfc91d32718] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=35.2 K 2024-11-07T12:37:37,062 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 95501280adff47508edfb1a40c3aa5cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730983053285 2024-11-07T12:37:37,063 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 655073c81c0548019951a2862cde9621, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730983053346 2024-11-07T12:37:37,063 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d9a4f901f349a6b9689bfc91d32718, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730983054513 2024-11-07T12:37:37,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742216_1392 (size=31058) 2024-11-07T12:37:37,083 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/1adc08a8e13b4a12a3cb95f3773595dc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/1adc08a8e13b4a12a3cb95f3773595dc 2024-11-07T12:37:37,084 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#C#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:37,085 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/8440b8c8ac074cef83ba7d50f94062d4 is 50, key is test_row_0/C:col10/1730983054513/Put/seqid=0 2024-11-07T12:37:37,089 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/A of 2e9c3a5f1845c325e45e01815b97b02f into 1adc08a8e13b4a12a3cb95f3773595dc(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:37,089 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:37,089 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/A, priority=13, startTime=1730983056581; duration=0sec 2024-11-07T12:37:37,090 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:37,090 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:A 2024-11-07T12:37:37,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:37,114 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fdf85afcd0ae494583178d0d8ae2cdc3_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdf85afcd0ae494583178d0d8ae2cdc3_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:37,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/a04c768edcad4c9cbaaa6f57f19c10df, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:37,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/a04c768edcad4c9cbaaa6f57f19c10df is 175, key is test_row_0/A:col10/1730983055683/Put/seqid=0 2024-11-07T12:37:37,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742217_1393 (size=12104) 2024-11-07T12:37:37,130 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/8440b8c8ac074cef83ba7d50f94062d4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/8440b8c8ac074cef83ba7d50f94062d4 2024-11-07T12:37:37,135 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/C of 2e9c3a5f1845c325e45e01815b97b02f into 8440b8c8ac074cef83ba7d50f94062d4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:37,135 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:37,135 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/C, priority=13, startTime=1730983056582; duration=0sec 2024-11-07T12:37:37,135 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:37,135 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:C 2024-11-07T12:37:37,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983117127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983117128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742218_1394 (size=30955) 2024-11-07T12:37:37,141 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/a04c768edcad4c9cbaaa6f57f19c10df 2024-11-07T12:37:37,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/be24aebfafe74704b0a7f545c9b5cb29 is 50, key is test_row_0/B:col10/1730983055683/Put/seqid=0 2024-11-07T12:37:37,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983117157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742219_1395 (size=12001) 2024-11-07T12:37:37,166 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/be24aebfafe74704b0a7f545c9b5cb29 2024-11-07T12:37:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/e12806e87e6c47b29adfe4925f068a90 is 50, key is test_row_0/C:col10/1730983055683/Put/seqid=0 2024-11-07T12:37:37,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742220_1396 (size=12001) 2024-11-07T12:37:37,182 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/e12806e87e6c47b29adfe4925f068a90 2024-11-07T12:37:37,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/a04c768edcad4c9cbaaa6f57f19c10df as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a04c768edcad4c9cbaaa6f57f19c10df 2024-11-07T12:37:37,194 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a04c768edcad4c9cbaaa6f57f19c10df, entries=150, sequenceid=76, filesize=30.2 K 2024-11-07T12:37:37,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/be24aebfafe74704b0a7f545c9b5cb29 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/be24aebfafe74704b0a7f545c9b5cb29 2024-11-07T12:37:37,201 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/be24aebfafe74704b0a7f545c9b5cb29, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T12:37:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/e12806e87e6c47b29adfe4925f068a90 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e12806e87e6c47b29adfe4925f068a90 2024-11-07T12:37:37,214 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e12806e87e6c47b29adfe4925f068a90, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T12:37:37,215 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2e9c3a5f1845c325e45e01815b97b02f in 537ms, sequenceid=76, compaction requested=false 2024-11-07T12:37:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-07T12:37:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-07T12:37:37,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-07T12:37:37,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7860 sec 2024-11-07T12:37:37,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.7960 sec 2024-11-07T12:37:37,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:37,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:37:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:37,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107da7e064bdbac4a43a15a07ad2d61593b_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983057449/Put/seqid=0 2024-11-07T12:37:37,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742221_1397 (size=14594) 2024-11-07T12:37:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:37:37,578 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:37,579 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-07T12:37:37,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:37:37,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:37,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983117605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983117614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983117614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:37:37,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T12:37:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:37,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:37,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:37,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983117737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983117738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983117741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T12:37:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:37,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:37:37,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983117897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983117897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,909 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in 2e9c3a5f1845c325e45e01815b97b02f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,909 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 2e9c3a5f1845c325e45e01815b97b02f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,915 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:37,922 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107da7e064bdbac4a43a15a07ad2d61593b_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da7e064bdbac4a43a15a07ad2d61593b_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:37,924 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/3a81b31a3ddc4fe8abec82026663ede4, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:37,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/3a81b31a3ddc4fe8abec82026663ede4 is 175, key is test_row_0/A:col10/1730983057449/Put/seqid=0 2024-11-07T12:37:37,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742222_1398 (size=39549) 2024-11-07T12:37:37,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983117957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/3a81b31a3ddc4fe8abec82026663ede4 2024-11-07T12:37:37,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983117968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983117968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:37,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/da384bb04df646dc9383c21cb30919a6 is 50, key is test_row_0/B:col10/1730983057449/Put/seqid=0 2024-11-07T12:37:38,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742223_1399 (size=12001) 2024-11-07T12:37:38,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/da384bb04df646dc9383c21cb30919a6 2024-11-07T12:37:38,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/f2c9592190db4a959cef40113aef89c4 is 50, key is test_row_0/C:col10/1730983057449/Put/seqid=0 2024-11-07T12:37:38,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742224_1400 (size=12001) 2024-11-07T12:37:38,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T12:37:38,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:38,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T12:37:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:37:38,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983118267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983118287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983118287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T12:37:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:38,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/f2c9592190db4a959cef40113aef89c4 2024-11-07T12:37:38,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/3a81b31a3ddc4fe8abec82026663ede4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/3a81b31a3ddc4fe8abec82026663ede4 2024-11-07T12:37:38,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/3a81b31a3ddc4fe8abec82026663ede4, entries=200, sequenceid=94, filesize=38.6 K 2024-11-07T12:37:38,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/da384bb04df646dc9383c21cb30919a6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/da384bb04df646dc9383c21cb30919a6 2024-11-07T12:37:38,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/da384bb04df646dc9383c21cb30919a6, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T12:37:38,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/f2c9592190db4a959cef40113aef89c4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/f2c9592190db4a959cef40113aef89c4 2024-11-07T12:37:38,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/f2c9592190db4a959cef40113aef89c4, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T12:37:38,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 2e9c3a5f1845c325e45e01815b97b02f in 1014ms, sequenceid=94, compaction requested=true 2024-11-07T12:37:38,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:38,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:38,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:38,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:38,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:37:38,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:38,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:37:38,466 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:38,466 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:38,467 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:38,467 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/C is initiating minor compaction (all files) 2024-11-07T12:37:38,467 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/C in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,467 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/8440b8c8ac074cef83ba7d50f94062d4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e12806e87e6c47b29adfe4925f068a90, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/f2c9592190db4a959cef40113aef89c4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=35.3 K 2024-11-07T12:37:38,467 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:38,467 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/A is initiating minor compaction (all files) 2024-11-07T12:37:38,468 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8440b8c8ac074cef83ba7d50f94062d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730983054513 2024-11-07T12:37:38,468 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/A in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,468 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/1adc08a8e13b4a12a3cb95f3773595dc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a04c768edcad4c9cbaaa6f57f19c10df, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/3a81b31a3ddc4fe8abec82026663ede4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=99.2 K 2024-11-07T12:37:38,468 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,468 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/1adc08a8e13b4a12a3cb95f3773595dc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a04c768edcad4c9cbaaa6f57f19c10df, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/3a81b31a3ddc4fe8abec82026663ede4] 2024-11-07T12:37:38,468 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e12806e87e6c47b29adfe4925f068a90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730983055639 2024-11-07T12:37:38,468 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1adc08a8e13b4a12a3cb95f3773595dc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730983054513 2024-11-07T12:37:38,469 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c9592190db4a959cef40113aef89c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983056782 2024-11-07T12:37:38,469 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a04c768edcad4c9cbaaa6f57f19c10df, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730983055639 2024-11-07T12:37:38,469 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a81b31a3ddc4fe8abec82026663ede4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983056782 2024-11-07T12:37:38,483 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#C#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:38,483 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/bff8593f651340778e0e6e04a315fd94 is 50, key is test_row_0/C:col10/1730983057449/Put/seqid=0 2024-11-07T12:37:38,492 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:38,499 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411078df7c7abbd004617a7124c57548b586b_2e9c3a5f1845c325e45e01815b97b02f store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:38,502 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411078df7c7abbd004617a7124c57548b586b_2e9c3a5f1845c325e45e01815b97b02f, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:38,502 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078df7c7abbd004617a7124c57548b586b_2e9c3a5f1845c325e45e01815b97b02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:38,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742225_1401 (size=12207) 2024-11-07T12:37:38,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T12:37:38,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,516 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:37:38,522 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/bff8593f651340778e0e6e04a315fd94 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bff8593f651340778e0e6e04a315fd94 2024-11-07T12:37:38,530 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/C of 2e9c3a5f1845c325e45e01815b97b02f into bff8593f651340778e0e6e04a315fd94(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:38,530 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:38,530 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/C, priority=13, startTime=1730983058465; duration=0sec 2024-11-07T12:37:38,530 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:38,530 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:C 2024-11-07T12:37:38,530 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:38,531 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:38,531 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/B is initiating minor compaction (all files) 2024-11-07T12:37:38,531 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/B in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:38,531 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/bf86d0f0e20644908e85aa2324fa1893, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/be24aebfafe74704b0a7f545c9b5cb29, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/da384bb04df646dc9383c21cb30919a6] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=35.3 K 2024-11-07T12:37:38,532 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bf86d0f0e20644908e85aa2324fa1893, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730983054513 2024-11-07T12:37:38,532 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting be24aebfafe74704b0a7f545c9b5cb29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730983055639 2024-11-07T12:37:38,532 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting da384bb04df646dc9383c21cb30919a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983056782 2024-11-07T12:37:38,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742226_1402 (size=4469) 2024-11-07T12:37:38,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071e158e785dab49c3a18dc7494f9b4455_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983057554/Put/seqid=0 2024-11-07T12:37:38,576 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#B#compaction#343 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:38,577 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/31790e697ff74399b71965eccc4152de is 50, key is test_row_0/B:col10/1730983057449/Put/seqid=0 2024-11-07T12:37:38,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742227_1403 (size=12154) 2024-11-07T12:37:38,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742228_1404 (size=12207) 2024-11-07T12:37:38,639 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/31790e697ff74399b71965eccc4152de as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/31790e697ff74399b71965eccc4152de 2024-11-07T12:37:38,647 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/B of 2e9c3a5f1845c325e45e01815b97b02f into 31790e697ff74399b71965eccc4152de(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:38,648 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:38,648 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/B, priority=13, startTime=1730983058465; duration=0sec 2024-11-07T12:37:38,648 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:38,648 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:B 2024-11-07T12:37:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:37:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:38,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:38,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983118840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983118840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983118856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,950 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#A#compaction#341 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:38,951 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/383aa54a63b04db7b67f6b530ce3ee8a is 175, key is test_row_0/A:col10/1730983057449/Put/seqid=0 2024-11-07T12:37:38,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742229_1405 (size=31161) 2024-11-07T12:37:38,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983118967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983118968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,979 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/383aa54a63b04db7b67f6b530ce3ee8a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/383aa54a63b04db7b67f6b530ce3ee8a 2024-11-07T12:37:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983118979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:38,987 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/A of 2e9c3a5f1845c325e45e01815b97b02f into 383aa54a63b04db7b67f6b530ce3ee8a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:38,987 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:38,987 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/A, priority=13, startTime=1730983058465; duration=0sec 2024-11-07T12:37:38,987 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:38,987 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:A 2024-11-07T12:37:39,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:39,035 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071e158e785dab49c3a18dc7494f9b4455_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071e158e785dab49c3a18dc7494f9b4455_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/9860b559b531453b81543a30ee4c324d, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/9860b559b531453b81543a30ee4c324d is 175, key is test_row_0/A:col10/1730983057554/Put/seqid=0 2024-11-07T12:37:39,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742230_1406 (size=30955) 2024-11-07T12:37:39,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983119177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:39,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983119177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:39,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983119197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:39,478 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/9860b559b531453b81543a30ee4c324d 2024-11-07T12:37:39,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983119487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/e4274471c5e8447181ff1291ad7e9387 is 50, key is test_row_0/B:col10/1730983057554/Put/seqid=0 2024-11-07T12:37:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:39,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983119499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983119517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:39,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742231_1407 (size=12001) 2024-11-07T12:37:39,547 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/e4274471c5e8447181ff1291ad7e9387 2024-11-07T12:37:39,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/a5efcf054e77493c96b1a0dcd11b9127 is 50, key is test_row_0/C:col10/1730983057554/Put/seqid=0 2024-11-07T12:37:39,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742232_1408 (size=12001) 2024-11-07T12:37:39,616 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/a5efcf054e77493c96b1a0dcd11b9127 2024-11-07T12:37:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/9860b559b531453b81543a30ee4c324d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9860b559b531453b81543a30ee4c324d 2024-11-07T12:37:39,632 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9860b559b531453b81543a30ee4c324d, entries=150, sequenceid=116, filesize=30.2 K 2024-11-07T12:37:39,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/e4274471c5e8447181ff1291ad7e9387 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e4274471c5e8447181ff1291ad7e9387 2024-11-07T12:37:39,636 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e4274471c5e8447181ff1291ad7e9387, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T12:37:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/a5efcf054e77493c96b1a0dcd11b9127 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a5efcf054e77493c96b1a0dcd11b9127 2024-11-07T12:37:39,648 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a5efcf054e77493c96b1a0dcd11b9127, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T12:37:39,649 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2e9c3a5f1845c325e45e01815b97b02f in 1133ms, sequenceid=116, compaction requested=false 2024-11-07T12:37:39,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:39,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:39,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-07T12:37:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-07T12:37:39,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-07T12:37:39,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0730 sec 2024-11-07T12:37:39,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.0780 sec 2024-11-07T12:37:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:37:39,738 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:39,738 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-07T12:37:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:37:39,741 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:39,741 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:39,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:37:39,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T12:37:39,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:39,897 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T12:37:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076751b4aa7d374657af3d4f1a77e15e97_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983058852/Put/seqid=0 2024-11-07T12:37:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:39,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742233_1409 (size=12204) 2024-11-07T12:37:39,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:39,947 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076751b4aa7d374657af3d4f1a77e15e97_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076751b4aa7d374657af3d4f1a77e15e97_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/004ddb04aa0a4d8e9247ddbd9b72720f, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/004ddb04aa0a4d8e9247ddbd9b72720f is 175, key is test_row_0/A:col10/1730983058852/Put/seqid=0 2024-11-07T12:37:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742234_1410 (size=31005) 2024-11-07T12:37:39,991 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/004ddb04aa0a4d8e9247ddbd9b72720f 2024-11-07T12:37:40,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/3b5f1e667f544623983660c277792fdc is 50, key is test_row_0/B:col10/1730983058852/Put/seqid=0 2024-11-07T12:37:40,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742235_1411 (size=12051) 2024-11-07T12:37:40,008 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/3b5f1e667f544623983660c277792fdc 2024-11-07T12:37:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/d90eb72bc0b645f88a3ec76fa3acaf89 is 50, key is test_row_0/C:col10/1730983058852/Put/seqid=0 2024-11-07T12:37:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742236_1412 (size=12051) 2024-11-07T12:37:40,039 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/d90eb72bc0b645f88a3ec76fa3acaf89 2024-11-07T12:37:40,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983120026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983120027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983120034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983120036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983120037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/004ddb04aa0a4d8e9247ddbd9b72720f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/004ddb04aa0a4d8e9247ddbd9b72720f 2024-11-07T12:37:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:37:40,062 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/004ddb04aa0a4d8e9247ddbd9b72720f, entries=150, sequenceid=133, filesize=30.3 K 2024-11-07T12:37:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/3b5f1e667f544623983660c277792fdc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b5f1e667f544623983660c277792fdc 2024-11-07T12:37:40,069 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b5f1e667f544623983660c277792fdc, entries=150, sequenceid=133, filesize=11.8 K 2024-11-07T12:37:40,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/d90eb72bc0b645f88a3ec76fa3acaf89 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d90eb72bc0b645f88a3ec76fa3acaf89 2024-11-07T12:37:40,078 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d90eb72bc0b645f88a3ec76fa3acaf89, entries=150, sequenceid=133, filesize=11.8 K 2024-11-07T12:37:40,081 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2e9c3a5f1845c325e45e01815b97b02f in 184ms, sequenceid=133, compaction requested=true 2024-11-07T12:37:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-07T12:37:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-07T12:37:40,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-07T12:37:40,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 341 msec 2024-11-07T12:37:40,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 347 msec 2024-11-07T12:37:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:40,160 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:40,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:37:40,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077867590b02ff42ec918938e6266ec690_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983060159/Put/seqid=0 2024-11-07T12:37:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983120178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983120180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742237_1413 (size=19774) 2024-11-07T12:37:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983120182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,195 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:40,200 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077867590b02ff42ec918938e6266ec690_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077867590b02ff42ec918938e6266ec690_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:40,200 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/b31b52637c5549e2be0afbb26eb3b78c, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:40,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/b31b52637c5549e2be0afbb26eb3b78c is 175, key is test_row_0/A:col10/1730983060159/Put/seqid=0 2024-11-07T12:37:40,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983120188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742238_1414 (size=57033) 2024-11-07T12:37:40,214 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/b31b52637c5549e2be0afbb26eb3b78c 2024-11-07T12:37:40,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/4f4b89b56b2949fcbc6fc094e7c61574 is 50, key is test_row_0/B:col10/1730983060159/Put/seqid=0 2024-11-07T12:37:40,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742239_1415 (size=12151) 2024-11-07T12:37:40,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983120298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983120298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983120298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983120307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:37:40,369 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:40,370 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-07T12:37:40,372 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:37:40,373 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:40,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:37:40,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:37:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:40,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983120529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983120530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983120530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983120530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/4f4b89b56b2949fcbc6fc094e7c61574 2024-11-07T12:37:40,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/a67fffd43f5040f1aaa6b36192463634 is 50, key is test_row_0/C:col10/1730983060159/Put/seqid=0 2024-11-07T12:37:40,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:37:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:37:40,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742240_1416 (size=12151) 2024-11-07T12:37:40,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/a67fffd43f5040f1aaa6b36192463634 2024-11-07T12:37:40,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/b31b52637c5549e2be0afbb26eb3b78c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/b31b52637c5549e2be0afbb26eb3b78c 2024-11-07T12:37:40,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/b31b52637c5549e2be0afbb26eb3b78c, entries=300, sequenceid=158, filesize=55.7 K 2024-11-07T12:37:40,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/4f4b89b56b2949fcbc6fc094e7c61574 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/4f4b89b56b2949fcbc6fc094e7c61574 2024-11-07T12:37:40,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/4f4b89b56b2949fcbc6fc094e7c61574, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T12:37:40,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/a67fffd43f5040f1aaa6b36192463634 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a67fffd43f5040f1aaa6b36192463634 2024-11-07T12:37:40,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a67fffd43f5040f1aaa6b36192463634, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T12:37:40,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2e9c3a5f1845c325e45e01815b97b02f in 583ms, sequenceid=158, compaction requested=true 2024-11-07T12:37:40,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:40,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:40,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:40,744 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:40,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:40,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:40,744 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:40,745 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:40,745 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/A is initiating minor compaction (all files) 2024-11-07T12:37:40,745 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/A in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,745 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:40,745 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/383aa54a63b04db7b67f6b530ce3ee8a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9860b559b531453b81543a30ee4c324d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/004ddb04aa0a4d8e9247ddbd9b72720f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/b31b52637c5549e2be0afbb26eb3b78c] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=146.6 K 2024-11-07T12:37:40,746 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/B is initiating minor compaction (all files) 2024-11-07T12:37:40,746 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,746 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/383aa54a63b04db7b67f6b530ce3ee8a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9860b559b531453b81543a30ee4c324d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/004ddb04aa0a4d8e9247ddbd9b72720f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/b31b52637c5549e2be0afbb26eb3b78c] 2024-11-07T12:37:40,746 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/B in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,746 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/31790e697ff74399b71965eccc4152de, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e4274471c5e8447181ff1291ad7e9387, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b5f1e667f544623983660c277792fdc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/4f4b89b56b2949fcbc6fc094e7c61574] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=47.3 K 2024-11-07T12:37:40,746 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 31790e697ff74399b71965eccc4152de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983056782 2024-11-07T12:37:40,746 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 383aa54a63b04db7b67f6b530ce3ee8a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983056782 2024-11-07T12:37:40,746 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e4274471c5e8447181ff1291ad7e9387, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730983057533 2024-11-07T12:37:40,747 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9860b559b531453b81543a30ee4c324d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730983057533 2024-11-07T12:37:40,747 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5f1e667f544623983660c277792fdc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730983058827 2024-11-07T12:37:40,747 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 004ddb04aa0a4d8e9247ddbd9b72720f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730983058827 2024-11-07T12:37:40,747 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f4b89b56b2949fcbc6fc094e7c61574, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983060030 2024-11-07T12:37:40,747 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b31b52637c5549e2be0afbb26eb3b78c, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983060011 2024-11-07T12:37:40,769 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:40,783 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#B#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:40,784 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/69b28ac5fadc4d9bb31c128408aa7b40 is 50, key is test_row_0/B:col10/1730983060159/Put/seqid=0 2024-11-07T12:37:40,785 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110786cb143ab6364257a5d76cf2ba39c430_2e9c3a5f1845c325e45e01815b97b02f store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:40,788 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110786cb143ab6364257a5d76cf2ba39c430_2e9c3a5f1845c325e45e01815b97b02f, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:40,788 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110786cb143ab6364257a5d76cf2ba39c430_2e9c3a5f1845c325e45e01815b97b02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:40,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:37:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,838 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:37:40,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742241_1417 (size=12493) 2024-11-07T12:37:40,854 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/69b28ac5fadc4d9bb31c128408aa7b40 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/69b28ac5fadc4d9bb31c128408aa7b40 2024-11-07T12:37:40,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742242_1418 (size=4469) 2024-11-07T12:37:40,857 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#A#compaction#352 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:40,858 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/589dd62547a745e7a0f5932e385f9c38 is 175, key is test_row_0/A:col10/1730983060159/Put/seqid=0 2024-11-07T12:37:40,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:40,864 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/B of 2e9c3a5f1845c325e45e01815b97b02f into 69b28ac5fadc4d9bb31c128408aa7b40(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:40,864 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:40,864 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/B, priority=12, startTime=1730983060744; duration=0sec 2024-11-07T12:37:40,864 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:40,864 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:B 2024-11-07T12:37:40,865 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:40,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107de52cf2f14284eaa96cc72d7de158f09_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983060180/Put/seqid=0 2024-11-07T12:37:40,868 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:40,868 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/C is initiating minor compaction (all files) 2024-11-07T12:37:40,869 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/C in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:40,869 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bff8593f651340778e0e6e04a315fd94, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a5efcf054e77493c96b1a0dcd11b9127, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d90eb72bc0b645f88a3ec76fa3acaf89, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a67fffd43f5040f1aaa6b36192463634] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=47.3 K 2024-11-07T12:37:40,871 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bff8593f651340778e0e6e04a315fd94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983056782 2024-11-07T12:37:40,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742243_1419 (size=31447) 2024-11-07T12:37:40,871 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a5efcf054e77493c96b1a0dcd11b9127, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730983057533 2024-11-07T12:37:40,873 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d90eb72bc0b645f88a3ec76fa3acaf89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730983058827 2024-11-07T12:37:40,873 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a67fffd43f5040f1aaa6b36192463634, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983060030 2024-11-07T12:37:40,877 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/589dd62547a745e7a0f5932e385f9c38 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/589dd62547a745e7a0f5932e385f9c38 2024-11-07T12:37:40,882 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/A of 2e9c3a5f1845c325e45e01815b97b02f into 589dd62547a745e7a0f5932e385f9c38(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:40,883 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:40,883 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/A, priority=12, startTime=1730983060744; duration=0sec 2024-11-07T12:37:40,883 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:40,883 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:A 2024-11-07T12:37:40,888 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#C#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:40,889 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/b682d9c8d3b148c38261a30d99a15534 is 50, key is test_row_0/C:col10/1730983060159/Put/seqid=0 2024-11-07T12:37:40,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742244_1420 (size=12304) 2024-11-07T12:37:40,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742245_1421 (size=12493) 2024-11-07T12:37:40,915 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/b682d9c8d3b148c38261a30d99a15534 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/b682d9c8d3b148c38261a30d99a15534 2024-11-07T12:37:40,926 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/C of 2e9c3a5f1845c325e45e01815b97b02f into b682d9c8d3b148c38261a30d99a15534(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:40,926 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:40,926 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/C, priority=12, startTime=1730983060744; duration=0sec 2024-11-07T12:37:40,926 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:40,926 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:C 2024-11-07T12:37:40,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983120940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983120947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983120954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983120956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:37:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983121068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983121068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983121068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,078 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 2e9c3a5f1845c325e45e01815b97b02f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983121077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983121077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:41,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983121287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983121287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983121287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983121287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,300 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107de52cf2f14284eaa96cc72d7de158f09_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107de52cf2f14284eaa96cc72d7de158f09_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:41,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/30300ded9d17414fbb8646020d4024c1, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:41,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/30300ded9d17414fbb8646020d4024c1 is 175, key is test_row_0/A:col10/1730983060180/Put/seqid=0 2024-11-07T12:37:41,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742246_1422 (size=31105) 2024-11-07T12:37:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:37:41,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983121609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983121609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983121609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983121609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:41,743 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/30300ded9d17414fbb8646020d4024c1 2024-11-07T12:37:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/e99104af086a4838ae826b0cfb4199bb is 50, key is test_row_0/B:col10/1730983060180/Put/seqid=0 2024-11-07T12:37:41,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742247_1423 (size=12151) 2024-11-07T12:37:41,782 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/e99104af086a4838ae826b0cfb4199bb 2024-11-07T12:37:41,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/ecec3cf83d2c4fdcbfc8adb9893ad49f is 50, key is test_row_0/C:col10/1730983060180/Put/seqid=0 2024-11-07T12:37:41,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742248_1424 (size=12151) 2024-11-07T12:37:41,813 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/ecec3cf83d2c4fdcbfc8adb9893ad49f 2024-11-07T12:37:41,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/30300ded9d17414fbb8646020d4024c1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/30300ded9d17414fbb8646020d4024c1 2024-11-07T12:37:41,826 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/30300ded9d17414fbb8646020d4024c1, entries=150, sequenceid=169, filesize=30.4 K 2024-11-07T12:37:41,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/e99104af086a4838ae826b0cfb4199bb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e99104af086a4838ae826b0cfb4199bb 2024-11-07T12:37:41,854 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e99104af086a4838ae826b0cfb4199bb, entries=150, sequenceid=169, filesize=11.9 K 2024-11-07T12:37:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/ecec3cf83d2c4fdcbfc8adb9893ad49f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/ecec3cf83d2c4fdcbfc8adb9893ad49f 2024-11-07T12:37:41,865 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/ecec3cf83d2c4fdcbfc8adb9893ad49f, entries=150, sequenceid=169, filesize=11.9 K 2024-11-07T12:37:41,866 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2e9c3a5f1845c325e45e01815b97b02f in 1028ms, sequenceid=169, compaction requested=false 2024-11-07T12:37:41,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:41,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:41,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-07T12:37:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-07T12:37:41,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-07T12:37:41,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-11-07T12:37:41,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5000 sec 2024-11-07T12:37:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:42,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:42,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:37:42,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983122135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983122137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983122141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983122142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078ca65759d1434087bbd3e3528cb8b3ad_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983062126/Put/seqid=0 2024-11-07T12:37:42,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742249_1425 (size=14794) 2024-11-07T12:37:42,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:42,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078ca65759d1434087bbd3e3528cb8b3ad_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ca65759d1434087bbd3e3528cb8b3ad_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:42,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/7eb8ed68794d406782550fd37a09f49c, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:42,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/7eb8ed68794d406782550fd37a09f49c is 175, key is test_row_0/A:col10/1730983062126/Put/seqid=0 2024-11-07T12:37:42,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983122247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983122258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983122258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983122258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742250_1426 (size=39749) 2024-11-07T12:37:42,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983122458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983122477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983122478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983122478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:37:42,518 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:42,519 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-07T12:37:42,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:37:42,522 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:42,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:37:42,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:37:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:42,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:42,679 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/7eb8ed68794d406782550fd37a09f49c 2024-11-07T12:37:42,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/544fd1879da24b63ace612966f171807 is 50, key is test_row_0/B:col10/1730983062126/Put/seqid=0 2024-11-07T12:37:42,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742251_1427 (size=12151) 2024-11-07T12:37:42,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983122777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983122798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983122798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983122798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:42,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:37:42,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:42,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:42,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:42,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:42,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:37:42,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:37:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:42,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:43,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983123088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,094 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 2e9c3a5f1845c325e45e01815b97b02f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/544fd1879da24b63ace612966f171807 2024-11-07T12:37:43,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/6bae048907fe4d80a02a2315c9053360 is 50, key is test_row_0/C:col10/1730983062126/Put/seqid=0 2024-11-07T12:37:43,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742252_1428 (size=12151) 2024-11-07T12:37:43,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/6bae048907fe4d80a02a2315c9053360 2024-11-07T12:37:43,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:37:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:43,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:43,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/7eb8ed68794d406782550fd37a09f49c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7eb8ed68794d406782550fd37a09f49c 2024-11-07T12:37:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:37:43,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7eb8ed68794d406782550fd37a09f49c, entries=200, sequenceid=199, filesize=38.8 K 2024-11-07T12:37:43,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/544fd1879da24b63ace612966f171807 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/544fd1879da24b63ace612966f171807 2024-11-07T12:37:43,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/544fd1879da24b63ace612966f171807, entries=150, sequenceid=199, filesize=11.9 K 2024-11-07T12:37:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/6bae048907fe4d80a02a2315c9053360 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/6bae048907fe4d80a02a2315c9053360 2024-11-07T12:37:43,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/6bae048907fe4d80a02a2315c9053360, entries=150, sequenceid=199, filesize=11.9 K 2024-11-07T12:37:43,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 2e9c3a5f1845c325e45e01815b97b02f in 1038ms, sequenceid=199, compaction requested=true 2024-11-07T12:37:43,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:43,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:43,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:43,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:43,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:43,164 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:43,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:43,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:43,164 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:43,167 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:43,167 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:43,167 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/A is initiating minor compaction (all files) 2024-11-07T12:37:43,167 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/B is initiating minor compaction (all files) 2024-11-07T12:37:43,167 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/A in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:43,167 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/B in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:43,167 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/589dd62547a745e7a0f5932e385f9c38, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/30300ded9d17414fbb8646020d4024c1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7eb8ed68794d406782550fd37a09f49c] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=99.9 K 2024-11-07T12:37:43,167 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/69b28ac5fadc4d9bb31c128408aa7b40, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e99104af086a4838ae826b0cfb4199bb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/544fd1879da24b63ace612966f171807] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=35.9 K 2024-11-07T12:37:43,167 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:43,167 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/589dd62547a745e7a0f5932e385f9c38, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/30300ded9d17414fbb8646020d4024c1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7eb8ed68794d406782550fd37a09f49c] 2024-11-07T12:37:43,167 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b28ac5fadc4d9bb31c128408aa7b40, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983060030 2024-11-07T12:37:43,167 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 589dd62547a745e7a0f5932e385f9c38, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983060030 2024-11-07T12:37:43,168 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e99104af086a4838ae826b0cfb4199bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1730983060176 2024-11-07T12:37:43,168 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30300ded9d17414fbb8646020d4024c1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1730983060176 2024-11-07T12:37:43,168 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 544fd1879da24b63ace612966f171807, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730983060935 2024-11-07T12:37:43,171 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eb8ed68794d406782550fd37a09f49c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730983060935 2024-11-07T12:37:43,177 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:43,182 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#B#compaction#362 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:43,183 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/e757c2f0287d40be9b2d76352c9dd4c1 is 50, key is test_row_0/B:col10/1730983062126/Put/seqid=0 2024-11-07T12:37:43,189 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110719a04a9291e74c42b0ecc945ed6c5141_2e9c3a5f1845c325e45e01815b97b02f store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:43,191 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110719a04a9291e74c42b0ecc945ed6c5141_2e9c3a5f1845c325e45e01815b97b02f, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:43,192 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110719a04a9291e74c42b0ecc945ed6c5141_2e9c3a5f1845c325e45e01815b97b02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:43,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742253_1429 (size=12595) 2024-11-07T12:37:43,233 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/e757c2f0287d40be9b2d76352c9dd4c1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e757c2f0287d40be9b2d76352c9dd4c1 2024-11-07T12:37:43,241 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/B of 2e9c3a5f1845c325e45e01815b97b02f into e757c2f0287d40be9b2d76352c9dd4c1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:43,241 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:43,241 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/B, priority=13, startTime=1730983063164; duration=0sec 2024-11-07T12:37:43,241 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:43,241 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:B 2024-11-07T12:37:43,241 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:43,244 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:43,244 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/C is initiating minor compaction (all files) 2024-11-07T12:37:43,244 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/C in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:43,244 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/b682d9c8d3b148c38261a30d99a15534, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/ecec3cf83d2c4fdcbfc8adb9893ad49f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/6bae048907fe4d80a02a2315c9053360] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=35.9 K 2024-11-07T12:37:43,245 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b682d9c8d3b148c38261a30d99a15534, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983060030 2024-11-07T12:37:43,246 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting ecec3cf83d2c4fdcbfc8adb9893ad49f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1730983060176 2024-11-07T12:37:43,247 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bae048907fe4d80a02a2315c9053360, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730983060935 2024-11-07T12:37:43,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742254_1430 (size=4469) 2024-11-07T12:37:43,265 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:43,266 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/003b26b8f6154a8d8cd47129d90839fd is 50, key is test_row_0/C:col10/1730983062126/Put/seqid=0 2024-11-07T12:37:43,267 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#A#compaction#361 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:43,268 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/a98e2b99afe442bf8afbee87c4e3428f is 175, key is test_row_0/A:col10/1730983062126/Put/seqid=0 2024-11-07T12:37:43,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:37:43,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:43,293 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T12:37:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:43,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742256_1432 (size=31549) 2024-11-07T12:37:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742255_1431 (size=12595) 2024-11-07T12:37:43,338 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/003b26b8f6154a8d8cd47129d90839fd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/003b26b8f6154a8d8cd47129d90839fd 2024-11-07T12:37:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107103c41df2fc840fea03529e238e3d2d9_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983062140/Put/seqid=0 2024-11-07T12:37:43,359 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/C of 2e9c3a5f1845c325e45e01815b97b02f into 003b26b8f6154a8d8cd47129d90839fd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:43,359 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:43,359 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/C, priority=13, startTime=1730983063164; duration=0sec 2024-11-07T12:37:43,359 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:43,359 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:C 2024-11-07T12:37:43,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742257_1433 (size=12304) 2024-11-07T12:37:43,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983123380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983123383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983123384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983123393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983123507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983123507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983123508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983123519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:37:43,730 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/a98e2b99afe442bf8afbee87c4e3428f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a98e2b99afe442bf8afbee87c4e3428f 2024-11-07T12:37:43,734 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/A of 2e9c3a5f1845c325e45e01815b97b02f into a98e2b99afe442bf8afbee87c4e3428f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:43,734 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:43,734 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/A, priority=13, startTime=1730983063164; duration=0sec 2024-11-07T12:37:43,735 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:43,735 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:A 2024-11-07T12:37:43,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983123728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983123728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983123728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983123728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:43,789 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107103c41df2fc840fea03529e238e3d2d9_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107103c41df2fc840fea03529e238e3d2d9_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/eeed016907ca4a968d50fab4728a2bd9, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/eeed016907ca4a968d50fab4728a2bd9 is 175, key is test_row_0/A:col10/1730983062140/Put/seqid=0 2024-11-07T12:37:43,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742258_1434 (size=31105) 2024-11-07T12:37:43,824 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/eeed016907ca4a968d50fab4728a2bd9 2024-11-07T12:37:43,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/765a60215129433a8205ba7cdca447f4 is 50, key is test_row_0/B:col10/1730983062140/Put/seqid=0 2024-11-07T12:37:43,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742259_1435 (size=12151) 2024-11-07T12:37:43,886 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/765a60215129433a8205ba7cdca447f4 2024-11-07T12:37:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/e45bcd5fa8844eb79ce5915fd47c8b7f is 50, key is test_row_0/C:col10/1730983062140/Put/seqid=0 2024-11-07T12:37:43,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742260_1436 (size=12151) 2024-11-07T12:37:43,927 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/e45bcd5fa8844eb79ce5915fd47c8b7f 2024-11-07T12:37:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/eeed016907ca4a968d50fab4728a2bd9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/eeed016907ca4a968d50fab4728a2bd9 2024-11-07T12:37:43,943 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/eeed016907ca4a968d50fab4728a2bd9, entries=150, sequenceid=209, filesize=30.4 K 2024-11-07T12:37:43,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/765a60215129433a8205ba7cdca447f4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/765a60215129433a8205ba7cdca447f4 2024-11-07T12:37:43,951 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/765a60215129433a8205ba7cdca447f4, entries=150, sequenceid=209, filesize=11.9 K 2024-11-07T12:37:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/e45bcd5fa8844eb79ce5915fd47c8b7f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e45bcd5fa8844eb79ce5915fd47c8b7f 2024-11-07T12:37:43,957 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e45bcd5fa8844eb79ce5915fd47c8b7f, entries=150, sequenceid=209, filesize=11.9 K 2024-11-07T12:37:43,957 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 2e9c3a5f1845c325e45e01815b97b02f in 664ms, sequenceid=209, compaction requested=false 2024-11-07T12:37:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-07T12:37:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-07T12:37:43,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-07T12:37:43,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4370 sec 2024-11-07T12:37:43,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4420 sec 2024-11-07T12:37:44,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:44,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T12:37:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:44,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110746529f90e7a44508aa7cceb3ed7cab48_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983063382/Put/seqid=0 2024-11-07T12:37:44,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983124058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983124059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742261_1437 (size=14794) 2024-11-07T12:37:44,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983124071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983124071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983124177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983124178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983124188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983124188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983124387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983124397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983124408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983124408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:44,481 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110746529f90e7a44508aa7cceb3ed7cab48_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110746529f90e7a44508aa7cceb3ed7cab48_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:44,483 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/82e30c6ac7a04a56ae7f9bbffc6e6937, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:44,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/82e30c6ac7a04a56ae7f9bbffc6e6937 is 175, key is test_row_0/A:col10/1730983063382/Put/seqid=0 2024-11-07T12:37:44,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742262_1438 (size=39749) 2024-11-07T12:37:44,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/82e30c6ac7a04a56ae7f9bbffc6e6937 2024-11-07T12:37:44,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/488bc81a33f14c248b84a7e624671e71 is 50, key is test_row_0/B:col10/1730983063382/Put/seqid=0 2024-11-07T12:37:44,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742263_1439 (size=12151) 2024-11-07T12:37:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:37:44,681 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:44,681 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-07T12:37:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:37:44,683 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:44,684 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:44,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:44,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983124697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983124717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983124728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983124729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:37:44,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:37:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:44,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:44,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/488bc81a33f14c248b84a7e624671e71 2024-11-07T12:37:44,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/288cf9285a8b4cbf896977e594e3c6c5 is 50, key is test_row_0/C:col10/1730983063382/Put/seqid=0 2024-11-07T12:37:44,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:37:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:44,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:37:45,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742264_1440 (size=12151) 2024-11-07T12:37:45,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:37:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:45,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983125218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:45,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983125238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:45,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983125247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:45,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983125247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:45,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:37:45,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:45,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:45,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:37:45,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/288cf9285a8b4cbf896977e594e3c6c5 2024-11-07T12:37:45,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/82e30c6ac7a04a56ae7f9bbffc6e6937 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/82e30c6ac7a04a56ae7f9bbffc6e6937 2024-11-07T12:37:45,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/82e30c6ac7a04a56ae7f9bbffc6e6937, entries=200, sequenceid=239, filesize=38.8 K 2024-11-07T12:37:45,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/488bc81a33f14c248b84a7e624671e71 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/488bc81a33f14c248b84a7e624671e71 2024-11-07T12:37:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/488bc81a33f14c248b84a7e624671e71, entries=150, sequenceid=239, filesize=11.9 K 2024-11-07T12:37:45,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/288cf9285a8b4cbf896977e594e3c6c5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/288cf9285a8b4cbf896977e594e3c6c5 2024-11-07T12:37:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/288cf9285a8b4cbf896977e594e3c6c5, entries=150, sequenceid=239, filesize=11.9 K 2024-11-07T12:37:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 2e9c3a5f1845c325e45e01815b97b02f in 1390ms, sequenceid=239, compaction requested=true 2024-11-07T12:37:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:45,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:45,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:45,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:45,443 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:37:45,443 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,444 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:45,444 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/A is initiating minor compaction (all files) 2024-11-07T12:37:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,444 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/A in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,444 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a98e2b99afe442bf8afbee87c4e3428f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/eeed016907ca4a968d50fab4728a2bd9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/82e30c6ac7a04a56ae7f9bbffc6e6937] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=100.0 K 2024-11-07T12:37:45,444 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,444 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a98e2b99afe442bf8afbee87c4e3428f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/eeed016907ca4a968d50fab4728a2bd9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/82e30c6ac7a04a56ae7f9bbffc6e6937] 2024-11-07T12:37:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,444 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,444 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/B is initiating minor compaction (all files) 2024-11-07T12:37:45,444 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/B in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,445 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e757c2f0287d40be9b2d76352c9dd4c1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/765a60215129433a8205ba7cdca447f4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/488bc81a33f14c248b84a7e624671e71] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=36.0 K 2024-11-07T12:37:45,445 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a98e2b99afe442bf8afbee87c4e3428f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730983060935 2024-11-07T12:37:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,445 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting eeed016907ca4a968d50fab4728a2bd9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730983062133 2024-11-07T12:37:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,445 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting e757c2f0287d40be9b2d76352c9dd4c1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730983060935 2024-11-07T12:37:45,445 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 765a60215129433a8205ba7cdca447f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730983062133 2024-11-07T12:37:45,445 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e30c6ac7a04a56ae7f9bbffc6e6937, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730983063371 2024-11-07T12:37:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,446 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 488bc81a33f14c248b84a7e624671e71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730983063371 2024-11-07T12:37:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:37:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,453 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T12:37:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,453 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,456 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110776298436c095475caf2eb18ee64d2bb7_2e9c3a5f1845c325e45e01815b97b02f store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,456 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#B#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,457 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/8ec0fd94bace4ddb8d6fd28d712bb39d is 50, key is test_row_0/B:col10/1730983063382/Put/seqid=0 2024-11-07T12:37:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,459 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110776298436c095475caf2eb18ee64d2bb7_2e9c3a5f1845c325e45e01815b97b02f, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,459 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110776298436c095475caf2eb18ee64d2bb7_2e9c3a5f1845c325e45e01815b97b02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070156722360934218a346cb84947c0924_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983064066/Put/seqid=0 2024-11-07T12:37:45,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742265_1441 (size=12697) 2024-11-07T12:37:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,480 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/8ec0fd94bace4ddb8d6fd28d712bb39d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8ec0fd94bace4ddb8d6fd28d712bb39d 2024-11-07T12:37:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,485 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/B of 2e9c3a5f1845c325e45e01815b97b02f into 8ec0fd94bace4ddb8d6fd28d712bb39d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:45,485 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:45,485 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/B, priority=13, startTime=1730983065442; duration=0sec 2024-11-07T12:37:45,485 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:45,485 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:B 2024-11-07T12:37:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,485 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,487 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,487 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/C is initiating minor compaction (all files) 2024-11-07T12:37:45,487 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/C in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,487 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/003b26b8f6154a8d8cd47129d90839fd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e45bcd5fa8844eb79ce5915fd47c8b7f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/288cf9285a8b4cbf896977e594e3c6c5] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=36.0 K 2024-11-07T12:37:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,487 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 003b26b8f6154a8d8cd47129d90839fd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730983060935 2024-11-07T12:37:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,488 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45bcd5fa8844eb79ce5915fd47c8b7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730983062133 2024-11-07T12:37:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,488 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 288cf9285a8b4cbf896977e594e3c6c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730983063371 2024-11-07T12:37:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742267_1443 (size=9814) 2024-11-07T12:37:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,494 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070156722360934218a346cb84947c0924_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070156722360934218a346cb84947c0924_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/5a38068958a443328e722ee64dcde594, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/5a38068958a443328e722ee64dcde594 is 175, key is test_row_0/A:col10/1730983064066/Put/seqid=0 2024-11-07T12:37:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,498 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#C#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742266_1442 (size=4469) 2024-11-07T12:37:45,498 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/0a508d734ded4c78a9584a1e0475462c is 50, key is test_row_0/C:col10/1730983063382/Put/seqid=0 2024-11-07T12:37:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,500 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#A#compaction#370 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,501 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/33053056679b4980808ee4f82031c136 is 175, key is test_row_0/A:col10/1730983063382/Put/seqid=0 2024-11-07T12:37:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742268_1444 (size=22461) 2024-11-07T12:37:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,527 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/5a38068958a443328e722ee64dcde594 2024-11-07T12:37:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742269_1445 (size=12697) 2024-11-07T12:37:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742270_1446 (size=31651) 2024-11-07T12:37:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/a621d91f64454109bf444ba90d3c5394 is 50, key is test_row_0/B:col10/1730983064066/Put/seqid=0 2024-11-07T12:37:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,540 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/0a508d734ded4c78a9584a1e0475462c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/0a508d734ded4c78a9584a1e0475462c 2024-11-07T12:37:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,544 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/C of 2e9c3a5f1845c325e45e01815b97b02f into 0a508d734ded4c78a9584a1e0475462c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:45,545 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:45,545 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/C, priority=13, startTime=1730983065443; duration=0sec 2024-11-07T12:37:45,545 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:45,545 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:C 2024-11-07T12:37:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,550 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/33053056679b4980808ee4f82031c136 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/33053056679b4980808ee4f82031c136 2024-11-07T12:37:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742271_1447 (size=9757) 2024-11-07T12:37:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,553 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/a621d91f64454109bf444ba90d3c5394 2024-11-07T12:37:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,557 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/A of 2e9c3a5f1845c325e45e01815b97b02f into 33053056679b4980808ee4f82031c136(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,557 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,557 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/A, priority=13, startTime=1730983065442; duration=0sec 2024-11-07T12:37:45,557 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:45,557 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:A 2024-11-07T12:37:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/936c13eac56a4f239c2eca123641544f is 50, key is test_row_0/C:col10/1730983064066/Put/seqid=0 2024-11-07T12:37:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742272_1448 (size=9757) 2024-11-07T12:37:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,595 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/936c13eac56a4f239c2eca123641544f 2024-11-07T12:37:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/5a38068958a443328e722ee64dcde594 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/5a38068958a443328e722ee64dcde594 2024-11-07T12:37:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,605 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/5a38068958a443328e722ee64dcde594, entries=100, sequenceid=247, filesize=21.9 K 2024-11-07T12:37:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/a621d91f64454109bf444ba90d3c5394 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/a621d91f64454109bf444ba90d3c5394 2024-11-07T12:37:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,612 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/a621d91f64454109bf444ba90d3c5394, entries=100, sequenceid=247, filesize=9.5 K 2024-11-07T12:37:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/936c13eac56a4f239c2eca123641544f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/936c13eac56a4f239c2eca123641544f 2024-11-07T12:37:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,619 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/936c13eac56a4f239c2eca123641544f, entries=100, sequenceid=247, filesize=9.5 K 2024-11-07T12:37:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,620 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2e9c3a5f1845c325e45e01815b97b02f in 167ms, sequenceid=247, compaction requested=false 2024-11-07T12:37:45,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:45,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-07T12:37:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-07T12:37:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-07T12:37:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 938 msec 2024-11-07T12:37:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 944 msec 2024-11-07T12:37:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:37:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,818 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,818 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-07T12:37:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,820 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:37:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,821 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:45,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:37:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T12:37:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:45,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-07T12:37:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-07T12:37:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-07T12:37:45,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-07T12:37:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 163 msec 2024-11-07T12:37:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:37:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,138 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,139 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-07T12:37:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:37:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,141 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,142 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:37:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T12:37:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-07T12:37:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-07T12:37:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-07T12:37:46,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-07T12:37:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 160 msec 2024-11-07T12:37:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,458 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,458 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-07T12:37:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:37:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,460 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,461 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:46,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:46,500 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:46,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:37:46,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073f5bea13043148cc91b6bedfce7c7c25_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983066481/Put/seqid=0 2024-11-07T12:37:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:37:46,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742273_1449 (size=17534) 2024-11-07T12:37:46,583 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:46,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073f5bea13043148cc91b6bedfce7c7c25_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f5bea13043148cc91b6bedfce7c7c25_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:46,591 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/ea184e2763ec42b980587372af6b1cad, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:46,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/ea184e2763ec42b980587372af6b1cad is 175, key is test_row_0/A:col10/1730983066481/Put/seqid=0 2024-11-07T12:37:46,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742274_1450 (size=48639) 2024-11-07T12:37:46,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T12:37:46,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:46,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:46,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:46,614 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/ea184e2763ec42b980587372af6b1cad 2024-11-07T12:37:46,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983126605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983126607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983126619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983126622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/6a0c6aeef2304f15a599438186033ba4 is 50, key is test_row_0/B:col10/1730983066481/Put/seqid=0 2024-11-07T12:37:46,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742275_1451 (size=12301) 2024-11-07T12:37:46,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/6a0c6aeef2304f15a599438186033ba4 2024-11-07T12:37:46,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/20afbbf08f704a5082e056cbe4163d9f is 50, key is test_row_0/C:col10/1730983066481/Put/seqid=0 2024-11-07T12:37:46,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742276_1452 (size=12301) 2024-11-07T12:37:46,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983126728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983126728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983126739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983126739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T12:37:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:46,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:37:46,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T12:37:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:46,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:46,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983126948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983126948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983126957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983126957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T12:37:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:47,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:37:47,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/20afbbf08f704a5082e056cbe4163d9f 2024-11-07T12:37:47,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/ea184e2763ec42b980587372af6b1cad as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/ea184e2763ec42b980587372af6b1cad 2024-11-07T12:37:47,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/ea184e2763ec42b980587372af6b1cad, entries=250, sequenceid=264, filesize=47.5 K 2024-11-07T12:37:47,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/6a0c6aeef2304f15a599438186033ba4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/6a0c6aeef2304f15a599438186033ba4 2024-11-07T12:37:47,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/6a0c6aeef2304f15a599438186033ba4, entries=150, sequenceid=264, filesize=12.0 K 2024-11-07T12:37:47,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/20afbbf08f704a5082e056cbe4163d9f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/20afbbf08f704a5082e056cbe4163d9f 2024-11-07T12:37:47,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59798 deadline: 1730983127128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,135 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 2e9c3a5f1845c325e45e01815b97b02f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:37:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/20afbbf08f704a5082e056cbe4163d9f, entries=150, sequenceid=264, filesize=12.0 K 2024-11-07T12:37:47,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2e9c3a5f1845c325e45e01815b97b02f in 640ms, sequenceid=264, compaction requested=true 2024-11-07T12:37:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:47,140 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:47,140 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:47,141 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:47,141 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:47,141 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/A is initiating minor compaction (all files) 2024-11-07T12:37:47,141 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/B is initiating minor compaction (all files) 2024-11-07T12:37:47,141 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/A in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:47,141 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/B in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:47,141 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/33053056679b4980808ee4f82031c136, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/5a38068958a443328e722ee64dcde594, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/ea184e2763ec42b980587372af6b1cad] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=100.3 K 2024-11-07T12:37:47,141 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8ec0fd94bace4ddb8d6fd28d712bb39d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/a621d91f64454109bf444ba90d3c5394, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/6a0c6aeef2304f15a599438186033ba4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=33.9 K 2024-11-07T12:37:47,141 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:47,141 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/33053056679b4980808ee4f82031c136, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/5a38068958a443328e722ee64dcde594, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/ea184e2763ec42b980587372af6b1cad] 2024-11-07T12:37:47,142 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ec0fd94bace4ddb8d6fd28d712bb39d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730983063371 2024-11-07T12:37:47,142 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33053056679b4980808ee4f82031c136, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730983063371 2024-11-07T12:37:47,143 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a621d91f64454109bf444ba90d3c5394, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730983064066 2024-11-07T12:37:47,143 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a38068958a443328e722ee64dcde594, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730983064066 2024-11-07T12:37:47,143 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a0c6aeef2304f15a599438186033ba4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983066452 2024-11-07T12:37:47,143 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea184e2763ec42b980587372af6b1cad, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983066451 2024-11-07T12:37:47,152 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:47,152 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:47,153 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/eed63dbb5d8649f890387ea094f4560b is 50, key is test_row_0/B:col10/1730983066481/Put/seqid=0 2024-11-07T12:37:47,156 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107ae168b1143074ec690820644ca71e2f7_2e9c3a5f1845c325e45e01815b97b02f store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:47,158 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107ae168b1143074ec690820644ca71e2f7_2e9c3a5f1845c325e45e01815b97b02f, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:47,158 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ae168b1143074ec690820644ca71e2f7_2e9c3a5f1845c325e45e01815b97b02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:47,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742277_1453 (size=12949) 2024-11-07T12:37:47,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742278_1454 (size=4469) 2024-11-07T12:37:47,174 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#A#compaction#380 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:47,175 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/cc05555ad28a447fbcdc015dd99a31a8 is 175, key is test_row_0/A:col10/1730983066481/Put/seqid=0 2024-11-07T12:37:47,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742279_1455 (size=31903) 2024-11-07T12:37:47,193 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/cc05555ad28a447fbcdc015dd99a31a8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/cc05555ad28a447fbcdc015dd99a31a8 2024-11-07T12:37:47,199 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/A of 2e9c3a5f1845c325e45e01815b97b02f into cc05555ad28a447fbcdc015dd99a31a8(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:47,199 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:47,199 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/A, priority=13, startTime=1730983067140; duration=0sec 2024-11-07T12:37:47,199 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:47,199 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:A 2024-11-07T12:37:47,199 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:47,204 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:47,204 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/C is initiating minor compaction (all files) 2024-11-07T12:37:47,204 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/C in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:47,204 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/0a508d734ded4c78a9584a1e0475462c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/936c13eac56a4f239c2eca123641544f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/20afbbf08f704a5082e056cbe4163d9f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=33.9 K 2024-11-07T12:37:47,204 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a508d734ded4c78a9584a1e0475462c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730983063371 2024-11-07T12:37:47,205 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936c13eac56a4f239c2eca123641544f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730983064066 2024-11-07T12:37:47,205 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20afbbf08f704a5082e056cbe4163d9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983066452 2024-11-07T12:37:47,215 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:47,216 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/69e7352a762a443fabdb838820820a2a is 50, key is test_row_0/C:col10/1730983066481/Put/seqid=0 2024-11-07T12:37:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742280_1456 (size=12949) 2024-11-07T12:37:47,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T12:37:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:47,229 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:37:47,239 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/69e7352a762a443fabdb838820820a2a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/69e7352a762a443fabdb838820820a2a 2024-11-07T12:37:47,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107854947e7aa974f76b0e4b4fa8f9c382d_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983066602/Put/seqid=0 2024-11-07T12:37:47,246 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/C of 2e9c3a5f1845c325e45e01815b97b02f into 69e7352a762a443fabdb838820820a2a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:47,246 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:47,246 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/C, priority=13, startTime=1730983067140; duration=0sec 2024-11-07T12:37:47,247 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:47,247 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:C 2024-11-07T12:37:47,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742281_1457 (size=12454) 2024-11-07T12:37:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:47,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983127280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983127288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983127289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983127290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983127398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983127407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983127408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983127409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,570 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/eed63dbb5d8649f890387ea094f4560b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/eed63dbb5d8649f890387ea094f4560b 2024-11-07T12:37:47,574 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/B of 2e9c3a5f1845c325e45e01815b97b02f into eed63dbb5d8649f890387ea094f4560b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:47,574 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:47,574 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/B, priority=13, startTime=1730983067140; duration=0sec 2024-11-07T12:37:47,575 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:47,575 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:B 2024-11-07T12:37:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:37:47,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983127618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983127628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983127628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983127628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:47,658 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107854947e7aa974f76b0e4b4fa8f9c382d_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107854947e7aa974f76b0e4b4fa8f9c382d_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:47,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/54ec44b8be264fe9b56f50dc5ce6c6e0, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:47,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/54ec44b8be264fe9b56f50dc5ce6c6e0 is 175, key is test_row_0/A:col10/1730983066602/Put/seqid=0 2024-11-07T12:37:47,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742282_1458 (size=31255) 2024-11-07T12:37:47,664 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/54ec44b8be264fe9b56f50dc5ce6c6e0 2024-11-07T12:37:47,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/3b658aad9a644a41ae998e07cc2933d0 is 50, key is test_row_0/B:col10/1730983066602/Put/seqid=0 2024-11-07T12:37:47,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742283_1459 (size=12301) 2024-11-07T12:37:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983127938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983127938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983127938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:47,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983127947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:48,083 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/3b658aad9a644a41ae998e07cc2933d0 2024-11-07T12:37:48,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/2cb1c76ff3364956b740eb2afac3d8f0 is 50, key is test_row_0/C:col10/1730983066602/Put/seqid=0 2024-11-07T12:37:48,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742284_1460 (size=12301) 2024-11-07T12:37:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983128448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983128457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:48,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983128457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:48,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983128467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:48,497 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/2cb1c76ff3364956b740eb2afac3d8f0 2024-11-07T12:37:48,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/54ec44b8be264fe9b56f50dc5ce6c6e0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/54ec44b8be264fe9b56f50dc5ce6c6e0 2024-11-07T12:37:48,504 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/54ec44b8be264fe9b56f50dc5ce6c6e0, entries=150, sequenceid=289, filesize=30.5 K 2024-11-07T12:37:48,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/3b658aad9a644a41ae998e07cc2933d0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b658aad9a644a41ae998e07cc2933d0 2024-11-07T12:37:48,509 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b658aad9a644a41ae998e07cc2933d0, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T12:37:48,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/2cb1c76ff3364956b740eb2afac3d8f0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/2cb1c76ff3364956b740eb2afac3d8f0 2024-11-07T12:37:48,515 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/2cb1c76ff3364956b740eb2afac3d8f0, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T12:37:48,516 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2e9c3a5f1845c325e45e01815b97b02f in 1287ms, sequenceid=289, compaction requested=false 2024-11-07T12:37:48,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:48,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:48,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-07T12:37:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-07T12:37:48,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-07T12:37:48,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0560 sec 2024-11-07T12:37:48,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.0650 sec 2024-11-07T12:37:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:37:48,608 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:48,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-07T12:37:48,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:37:48,611 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:48,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:37:48,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T12:37:48,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:48,764 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:37:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110757769905b20d44cdb0fd7b62f234f5ef_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983067288/Put/seqid=0 2024-11-07T12:37:48,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742285_1461 (size=12454) 2024-11-07T12:37:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:37:49,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:49,181 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110757769905b20d44cdb0fd7b62f234f5ef_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757769905b20d44cdb0fd7b62f234f5ef_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/6776bf4c5ce14d92a6c5ea1386290c39, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:49,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/6776bf4c5ce14d92a6c5ea1386290c39 is 175, key is test_row_0/A:col10/1730983067288/Put/seqid=0 2024-11-07T12:37:49,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742286_1462 (size=31255) 2024-11-07T12:37:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:37:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:49,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983129521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983129524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983129526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983129528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,595 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/6776bf4c5ce14d92a6c5ea1386290c39 2024-11-07T12:37:49,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/61d7c99cef91497bb5ebde87e8fc6e11 is 50, key is test_row_0/B:col10/1730983067288/Put/seqid=0 2024-11-07T12:37:49,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742287_1463 (size=12301) 2024-11-07T12:37:49,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983129638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983129647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983129647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983129647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:37:49,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983129857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983129857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983129857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:49,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983129867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,009 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/61d7c99cef91497bb5ebde87e8fc6e11 2024-11-07T12:37:50,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/bd6827ed84724adfb0f11bb22e5c7d4d is 50, key is test_row_0/C:col10/1730983067288/Put/seqid=0 2024-11-07T12:37:50,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742288_1464 (size=12301) 2024-11-07T12:37:50,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983130178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983130179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983130179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983130187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,423 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/bd6827ed84724adfb0f11bb22e5c7d4d 2024-11-07T12:37:50,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/6776bf4c5ce14d92a6c5ea1386290c39 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/6776bf4c5ce14d92a6c5ea1386290c39 2024-11-07T12:37:50,436 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/6776bf4c5ce14d92a6c5ea1386290c39, entries=150, sequenceid=303, filesize=30.5 K 2024-11-07T12:37:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/61d7c99cef91497bb5ebde87e8fc6e11 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/61d7c99cef91497bb5ebde87e8fc6e11 2024-11-07T12:37:50,441 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/61d7c99cef91497bb5ebde87e8fc6e11, entries=150, sequenceid=303, filesize=12.0 K 2024-11-07T12:37:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/bd6827ed84724adfb0f11bb22e5c7d4d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bd6827ed84724adfb0f11bb22e5c7d4d 2024-11-07T12:37:50,448 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bd6827ed84724adfb0f11bb22e5c7d4d, entries=150, sequenceid=303, filesize=12.0 K 2024-11-07T12:37:50,449 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2e9c3a5f1845c325e45e01815b97b02f in 1685ms, sequenceid=303, compaction requested=true 2024-11-07T12:37:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:50,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-07T12:37:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-07T12:37:50,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-07T12:37:50,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-11-07T12:37:50,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8480 sec 2024-11-07T12:37:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:50,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:50,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:37:50,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110722e59189abe44c1296837098627f019f_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983069495/Put/seqid=0 2024-11-07T12:37:50,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742289_1465 (size=14994) 2024-11-07T12:37:50,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983130721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983130721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983130723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983130725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:37:50,757 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:50,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:37:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-07T12:37:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:37:50,759 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:37:50,760 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:37:50,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:37:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983130837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983130837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983130847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983130847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:37:50,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:37:50,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:50,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:50,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:50,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:50,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983131047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983131047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983131058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983131058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:37:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:37:51,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:51,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110722e59189abe44c1296837098627f019f_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110722e59189abe44c1296837098627f019f_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:51,130 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/36afe3f74abc433c8bfcf2e897a568e1, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:51,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/36afe3f74abc433c8bfcf2e897a568e1 is 175, key is test_row_0/A:col10/1730983069495/Put/seqid=0 2024-11-07T12:37:51,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742290_1466 (size=39949) 2024-11-07T12:37:51,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:37:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983131357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983131358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:37:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983131367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983131377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:37:51,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:37:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:51,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:37:51,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/36afe3f74abc433c8bfcf2e897a568e1 2024-11-07T12:37:51,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/9624b7b9b6324cf5b06774a8b3fc252d is 50, key is test_row_0/B:col10/1730983069495/Put/seqid=0 2024-11-07T12:37:51,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742291_1467 (size=12301) 2024-11-07T12:37:51,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/9624b7b9b6324cf5b06774a8b3fc252d 2024-11-07T12:37:51,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/25f2c4a651e24dbe843a7776b074ecab is 50, key is test_row_0/C:col10/1730983069495/Put/seqid=0 2024-11-07T12:37:51,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742292_1468 (size=12301) 2024-11-07T12:37:51,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/25f2c4a651e24dbe843a7776b074ecab 2024-11-07T12:37:51,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/36afe3f74abc433c8bfcf2e897a568e1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36afe3f74abc433c8bfcf2e897a568e1 2024-11-07T12:37:51,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36afe3f74abc433c8bfcf2e897a568e1, entries=200, sequenceid=328, filesize=39.0 K 2024-11-07T12:37:51,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/9624b7b9b6324cf5b06774a8b3fc252d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/9624b7b9b6324cf5b06774a8b3fc252d 2024-11-07T12:37:51,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/9624b7b9b6324cf5b06774a8b3fc252d, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T12:37:51,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/25f2c4a651e24dbe843a7776b074ecab as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/25f2c4a651e24dbe843a7776b074ecab 2024-11-07T12:37:51,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/25f2c4a651e24dbe843a7776b074ecab, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T12:37:51,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2e9c3a5f1845c325e45e01815b97b02f in 890ms, sequenceid=328, compaction requested=true 2024-11-07T12:37:51,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:51,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:51,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:51,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:51,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:51,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:51,594 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:51,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:51,594 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:51,595 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:51,595 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:51,595 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/A is initiating minor compaction (all files) 2024-11-07T12:37:51,595 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/B is initiating minor compaction (all files) 2024-11-07T12:37:51,595 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/A in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,595 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/B in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,595 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/cc05555ad28a447fbcdc015dd99a31a8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/54ec44b8be264fe9b56f50dc5ce6c6e0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/6776bf4c5ce14d92a6c5ea1386290c39, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36afe3f74abc433c8bfcf2e897a568e1] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=131.2 K 2024-11-07T12:37:51,595 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/eed63dbb5d8649f890387ea094f4560b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b658aad9a644a41ae998e07cc2933d0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/61d7c99cef91497bb5ebde87e8fc6e11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/9624b7b9b6324cf5b06774a8b3fc252d] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=48.7 K 2024-11-07T12:37:51,595 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,595 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/cc05555ad28a447fbcdc015dd99a31a8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/54ec44b8be264fe9b56f50dc5ce6c6e0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/6776bf4c5ce14d92a6c5ea1386290c39, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36afe3f74abc433c8bfcf2e897a568e1] 2024-11-07T12:37:51,595 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting eed63dbb5d8649f890387ea094f4560b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983066452 2024-11-07T12:37:51,596 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc05555ad28a447fbcdc015dd99a31a8, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983066452 2024-11-07T12:37:51,596 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b658aad9a644a41ae998e07cc2933d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983066602 2024-11-07T12:37:51,596 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54ec44b8be264fe9b56f50dc5ce6c6e0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983066602 2024-11-07T12:37:51,596 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 61d7c99cef91497bb5ebde87e8fc6e11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1730983067279 2024-11-07T12:37:51,596 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6776bf4c5ce14d92a6c5ea1386290c39, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1730983067279 2024-11-07T12:37:51,597 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9624b7b9b6324cf5b06774a8b3fc252d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983069495 2024-11-07T12:37:51,597 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36afe3f74abc433c8bfcf2e897a568e1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983069495 2024-11-07T12:37:51,607 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#B#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:51,607 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/2a255722a8184bfa877891af956ee299 is 50, key is test_row_0/B:col10/1730983069495/Put/seqid=0 2024-11-07T12:37:51,621 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:51,626 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411072ed05bb78a6546ec9733642700357dfb_2e9c3a5f1845c325e45e01815b97b02f store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:51,627 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411072ed05bb78a6546ec9733642700357dfb_2e9c3a5f1845c325e45e01815b97b02f, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:51,628 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072ed05bb78a6546ec9733642700357dfb_2e9c3a5f1845c325e45e01815b97b02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:51,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742293_1469 (size=13085) 2024-11-07T12:37:51,644 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/2a255722a8184bfa877891af956ee299 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/2a255722a8184bfa877891af956ee299 2024-11-07T12:37:51,651 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/B of 2e9c3a5f1845c325e45e01815b97b02f into 2a255722a8184bfa877891af956ee299(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:51,651 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:51,651 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/B, priority=12, startTime=1730983071594; duration=0sec 2024-11-07T12:37:51,651 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:51,651 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:B 2024-11-07T12:37:51,651 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:37:51,652 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:37:51,652 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/C is initiating minor compaction (all files) 2024-11-07T12:37:51,652 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/C in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,652 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/69e7352a762a443fabdb838820820a2a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/2cb1c76ff3364956b740eb2afac3d8f0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bd6827ed84724adfb0f11bb22e5c7d4d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/25f2c4a651e24dbe843a7776b074ecab] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=48.7 K 2024-11-07T12:37:51,653 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 69e7352a762a443fabdb838820820a2a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983066452 2024-11-07T12:37:51,655 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb1c76ff3364956b740eb2afac3d8f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983066602 2024-11-07T12:37:51,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742294_1470 (size=4469) 2024-11-07T12:37:51,656 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bd6827ed84724adfb0f11bb22e5c7d4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1730983067279 2024-11-07T12:37:51,657 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f2c4a651e24dbe843a7776b074ecab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983069495 2024-11-07T12:37:51,665 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:51,666 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/524c25cb7856474aa9844f958b9fd97e is 50, key is test_row_0/C:col10/1730983069495/Put/seqid=0 2024-11-07T12:37:51,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:37:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:51,680 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:37:51,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742295_1471 (size=13085) 2024-11-07T12:37:51,687 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/524c25cb7856474aa9844f958b9fd97e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/524c25cb7856474aa9844f958b9fd97e 2024-11-07T12:37:51,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e6168ba715f5487c92c7dceb7de63990_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983070720/Put/seqid=0 2024-11-07T12:37:51,691 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/C of 2e9c3a5f1845c325e45e01815b97b02f into 524c25cb7856474aa9844f958b9fd97e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:51,691 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:51,691 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/C, priority=12, startTime=1730983071594; duration=0sec 2024-11-07T12:37:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742296_1472 (size=12454) 2024-11-07T12:37:51,691 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:51,691 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:C 2024-11-07T12:37:51,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:51,695 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e6168ba715f5487c92c7dceb7de63990_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6168ba715f5487c92c7dceb7de63990_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/20dbe3cd84814d8bbd11237567ce53ca, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/20dbe3cd84814d8bbd11237567ce53ca is 175, key is test_row_0/A:col10/1730983070720/Put/seqid=0 2024-11-07T12:37:51,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742297_1473 (size=31255) 2024-11-07T12:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:51,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. as already flushing 2024-11-07T12:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:37:51,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983131929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983131940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983131941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:51,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983131943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,057 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#A#compaction#392 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983132047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,058 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/01c69e27db3a4c678308b317a9c0faf5 is 175, key is test_row_0/A:col10/1730983069495/Put/seqid=0 2024-11-07T12:37:52,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983132047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742298_1474 (size=32039) 2024-11-07T12:37:52,066 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/01c69e27db3a4c678308b317a9c0faf5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/01c69e27db3a4c678308b317a9c0faf5 2024-11-07T12:37:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983132057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983132057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,071 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/A of 2e9c3a5f1845c325e45e01815b97b02f into 01c69e27db3a4c678308b317a9c0faf5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:52,071 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:52,071 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/A, priority=12, startTime=1730983071593; duration=0sec 2024-11-07T12:37:52,072 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:52,072 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:A 2024-11-07T12:37:52,101 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/20dbe3cd84814d8bbd11237567ce53ca 2024-11-07T12:37:52,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/1e21426479ca43f18a4d53387c47d49f is 50, key is test_row_0/B:col10/1730983070720/Put/seqid=0 2024-11-07T12:37:52,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742299_1475 (size=12301) 2024-11-07T12:37:52,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983132267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983132267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983132277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983132277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,524 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/1e21426479ca43f18a4d53387c47d49f 2024-11-07T12:37:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/d9a1812d8fbb460892fcd623bbbeb3a6 is 50, key is test_row_0/C:col10/1730983070720/Put/seqid=0 2024-11-07T12:37:52,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742300_1476 (size=12301) 2024-11-07T12:37:52,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983132578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983132587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983132597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983132597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:37:52,938 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/d9a1812d8fbb460892fcd623bbbeb3a6 2024-11-07T12:37:52,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/20dbe3cd84814d8bbd11237567ce53ca as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/20dbe3cd84814d8bbd11237567ce53ca 2024-11-07T12:37:52,946 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/20dbe3cd84814d8bbd11237567ce53ca, entries=150, sequenceid=340, filesize=30.5 K 2024-11-07T12:37:52,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/1e21426479ca43f18a4d53387c47d49f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/1e21426479ca43f18a4d53387c47d49f 2024-11-07T12:37:52,950 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/1e21426479ca43f18a4d53387c47d49f, entries=150, sequenceid=340, filesize=12.0 K 2024-11-07T12:37:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/d9a1812d8fbb460892fcd623bbbeb3a6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d9a1812d8fbb460892fcd623bbbeb3a6 2024-11-07T12:37:52,954 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d9a1812d8fbb460892fcd623bbbeb3a6, entries=150, sequenceid=340, filesize=12.0 K 2024-11-07T12:37:52,955 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2e9c3a5f1845c325e45e01815b97b02f in 1275ms, sequenceid=340, compaction requested=false 2024-11-07T12:37:52,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:52,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:52,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-07T12:37:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-07T12:37:52,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-07T12:37:52,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1960 sec 2024-11-07T12:37:52,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.2010 sec 2024-11-07T12:37:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:53,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:37:53,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f19cfc360ae44f548a3a99c6af31d891_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983071938/Put/seqid=0 2024-11-07T12:37:53,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742301_1477 (size=14994) 2024-11-07T12:37:53,114 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:53,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f19cfc360ae44f548a3a99c6af31d891_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f19cfc360ae44f548a3a99c6af31d891_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:53,119 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/03aba7b4b8824e9eac8a7d500897da59, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:53,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/03aba7b4b8824e9eac8a7d500897da59 is 175, key is test_row_0/A:col10/1730983071938/Put/seqid=0 2024-11-07T12:37:53,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742302_1478 (size=39949) 2024-11-07T12:37:53,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983133117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983133122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983133124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983133126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983133237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983133237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983133237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983133237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,282 INFO [Thread-1702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1702. 2024-11-07T12:37:53,282 DEBUG [Thread-1702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:53,282 DEBUG [Thread-1702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,282 DEBUG [Thread-1702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:53,284 INFO [Thread-1700 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1700. 2024-11-07T12:37:53,284 DEBUG [Thread-1700 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:53,285 DEBUG [Thread-1700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,285 DEBUG [Thread-1700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:53,290 INFO [Thread-1704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1704. 2024-11-07T12:37:53,290 DEBUG [Thread-1704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:53,290 DEBUG [Thread-1704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,290 DEBUG [Thread-1704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:53,291 INFO [Thread-1708 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1708. 2024-11-07T12:37:53,291 DEBUG [Thread-1708 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:53,291 DEBUG [Thread-1708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,291 DEBUG [Thread-1708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:53,294 INFO [Thread-1706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1706. 2024-11-07T12:37:53,294 DEBUG [Thread-1706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:53,294 DEBUG [Thread-1706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,294 DEBUG [Thread-1706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:53,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983133448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983133448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983133448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983133448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,526 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/03aba7b4b8824e9eac8a7d500897da59 2024-11-07T12:37:53,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/8f5ae949222546b09bc74617150aa96f is 50, key is test_row_0/B:col10/1730983071938/Put/seqid=0 2024-11-07T12:37:53,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742303_1479 (size=12301) 2024-11-07T12:37:53,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983133757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983133757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983133757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983133757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:53,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/8f5ae949222546b09bc74617150aa96f 2024-11-07T12:37:53,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/109e8259327f4183b1b95a8524c13ecb is 50, key is test_row_0/C:col10/1730983071938/Put/seqid=0 2024-11-07T12:37:53,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742304_1480 (size=12301) 2024-11-07T12:37:54,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:54,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:54,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1730983134267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59854 deadline: 1730983134267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1730983134267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59866 deadline: 1730983134267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:37:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:37:54,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/109e8259327f4183b1b95a8524c13ecb 2024-11-07T12:37:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/03aba7b4b8824e9eac8a7d500897da59 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/03aba7b4b8824e9eac8a7d500897da59 2024-11-07T12:37:54,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/03aba7b4b8824e9eac8a7d500897da59, entries=200, sequenceid=368, filesize=39.0 K 2024-11-07T12:37:54,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/8f5ae949222546b09bc74617150aa96f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8f5ae949222546b09bc74617150aa96f 2024-11-07T12:37:54,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8f5ae949222546b09bc74617150aa96f, entries=150, sequenceid=368, filesize=12.0 K 2024-11-07T12:37:54,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/109e8259327f4183b1b95a8524c13ecb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/109e8259327f4183b1b95a8524c13ecb 2024-11-07T12:37:54,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/109e8259327f4183b1b95a8524c13ecb, entries=150, sequenceid=368, filesize=12.0 K 2024-11-07T12:37:54,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2e9c3a5f1845c325e45e01815b97b02f in 1257ms, sequenceid=368, compaction requested=true 2024-11-07T12:37:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:37:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:54,358 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:37:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2e9c3a5f1845c325e45e01815b97b02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:37:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:54,358 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:54,359 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:54,359 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:54,359 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/A is initiating minor compaction (all files) 2024-11-07T12:37:54,359 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/B is initiating minor compaction (all files) 2024-11-07T12:37:54,359 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/B in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:54,359 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/A in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:54,359 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/01c69e27db3a4c678308b317a9c0faf5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/20dbe3cd84814d8bbd11237567ce53ca, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/03aba7b4b8824e9eac8a7d500897da59] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=100.8 K 2024-11-07T12:37:54,359 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/2a255722a8184bfa877891af956ee299, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/1e21426479ca43f18a4d53387c47d49f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8f5ae949222546b09bc74617150aa96f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=36.8 K 2024-11-07T12:37:54,359 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:54,359 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/01c69e27db3a4c678308b317a9c0faf5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/20dbe3cd84814d8bbd11237567ce53ca, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/03aba7b4b8824e9eac8a7d500897da59] 2024-11-07T12:37:54,359 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a255722a8184bfa877891af956ee299, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983069495 2024-11-07T12:37:54,359 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c69e27db3a4c678308b317a9c0faf5, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983069495 2024-11-07T12:37:54,360 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e21426479ca43f18a4d53387c47d49f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730983070718 2024-11-07T12:37:54,360 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20dbe3cd84814d8bbd11237567ce53ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730983070718 2024-11-07T12:37:54,360 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03aba7b4b8824e9eac8a7d500897da59, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1730983071938 2024-11-07T12:37:54,360 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5ae949222546b09bc74617150aa96f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1730983071938 2024-11-07T12:37:54,365 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:54,366 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:54,367 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/7dced02d315f4af8b1b82abd301303af is 50, key is test_row_0/B:col10/1730983071938/Put/seqid=0 2024-11-07T12:37:54,367 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411071364227c28a74f4aa4c76d288c8d9b9c_2e9c3a5f1845c325e45e01815b97b02f store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:54,369 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411071364227c28a74f4aa4c76d288c8d9b9c_2e9c3a5f1845c325e45e01815b97b02f, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:54,370 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071364227c28a74f4aa4c76d288c8d9b9c_2e9c3a5f1845c325e45e01815b97b02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:54,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742305_1481 (size=13187) 2024-11-07T12:37:54,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742306_1482 (size=4469) 2024-11-07T12:37:54,778 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/7dced02d315f4af8b1b82abd301303af as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/7dced02d315f4af8b1b82abd301303af 2024-11-07T12:37:54,782 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/B of 2e9c3a5f1845c325e45e01815b97b02f into 7dced02d315f4af8b1b82abd301303af(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:54,782 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:54,782 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/B, priority=13, startTime=1730983074358; duration=0sec 2024-11-07T12:37:54,782 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:37:54,782 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:B 2024-11-07T12:37:54,782 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:37:54,783 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:37:54,783 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 2e9c3a5f1845c325e45e01815b97b02f/C is initiating minor compaction (all files) 2024-11-07T12:37:54,783 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2e9c3a5f1845c325e45e01815b97b02f/C in TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:54,783 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/524c25cb7856474aa9844f958b9fd97e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d9a1812d8fbb460892fcd623bbbeb3a6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/109e8259327f4183b1b95a8524c13ecb] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp, totalSize=36.8 K 2024-11-07T12:37:54,783 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#A#compaction#401 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:54,783 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 524c25cb7856474aa9844f958b9fd97e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983069495 2024-11-07T12:37:54,783 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a1812d8fbb460892fcd623bbbeb3a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730983070718 2024-11-07T12:37:54,784 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/70d5d43fe4d7457dbde0edf25b39ee11 is 175, key is test_row_0/A:col10/1730983071938/Put/seqid=0 2024-11-07T12:37:54,784 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 109e8259327f4183b1b95a8524c13ecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1730983071938 2024-11-07T12:37:54,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742307_1483 (size=32141) 2024-11-07T12:37:54,790 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2e9c3a5f1845c325e45e01815b97b02f#C#compaction#402 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:37:54,790 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/3ff5da3db4aa434f8733e7dc496d3f6c is 50, key is test_row_0/C:col10/1730983071938/Put/seqid=0 2024-11-07T12:37:54,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742308_1484 (size=13187) 2024-11-07T12:37:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:37:54,928 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:55,191 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/70d5d43fe4d7457dbde0edf25b39ee11 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/70d5d43fe4d7457dbde0edf25b39ee11 2024-11-07T12:37:55,195 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/A of 2e9c3a5f1845c325e45e01815b97b02f into 70d5d43fe4d7457dbde0edf25b39ee11(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:55,195 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:55,195 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/A, priority=13, startTime=1730983074358; duration=0sec 2024-11-07T12:37:55,195 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:55,195 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:A 2024-11-07T12:37:55,196 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/3ff5da3db4aa434f8733e7dc496d3f6c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/3ff5da3db4aa434f8733e7dc496d3f6c 2024-11-07T12:37:55,200 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2e9c3a5f1845c325e45e01815b97b02f/C of 2e9c3a5f1845c325e45e01815b97b02f into 3ff5da3db4aa434f8733e7dc496d3f6c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:37:55,200 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:55,200 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., storeName=2e9c3a5f1845c325e45e01815b97b02f/C, priority=13, startTime=1730983074358; duration=0sec 2024-11-07T12:37:55,200 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:37:55,200 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2e9c3a5f1845c325e45e01815b97b02f:C 2024-11-07T12:37:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:55,279 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:37:55,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:37:55,280 INFO [Thread-1689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1689. 2024-11-07T12:37:55,280 INFO [Thread-1693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1693. 2024-11-07T12:37:55,280 DEBUG [Thread-1693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:55,280 DEBUG [Thread-1689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:55,280 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:55,280 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:55,280 INFO [Thread-1691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1691. 2024-11-07T12:37:55,280 DEBUG [Thread-1691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:55,280 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:55,280 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:55,280 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:55,280 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:55,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:55,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:55,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:55,281 INFO [Thread-1695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1695. 2024-11-07T12:37:55,281 DEBUG [Thread-1695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:55,281 DEBUG [Thread-1695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:55,281 DEBUG [Thread-1695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:55,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:55,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110798573471a9114dc39cc5bb7c5301b870_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_0/A:col10/1730983075278/Put/seqid=0 2024-11-07T12:37:55,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742309_1485 (size=12454) 2024-11-07T12:37:55,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:55,291 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110798573471a9114dc39cc5bb7c5301b870_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110798573471a9114dc39cc5bb7c5301b870_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:55,292 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/7506e8bc41af42db90e381206b4526ae, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:55,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/7506e8bc41af42db90e381206b4526ae is 175, key is test_row_0/A:col10/1730983075278/Put/seqid=0 2024-11-07T12:37:55,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742310_1486 (size=31255) 2024-11-07T12:37:55,696 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/7506e8bc41af42db90e381206b4526ae 2024-11-07T12:37:55,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/04d33b71059b4797be53b21d161b2f6b is 50, key is test_row_0/B:col10/1730983075278/Put/seqid=0 2024-11-07T12:37:55,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742311_1487 (size=12301) 2024-11-07T12:37:56,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/04d33b71059b4797be53b21d161b2f6b 2024-11-07T12:37:56,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/557b7482b9ce46bea6501e4b4b0cbe57 is 50, key is test_row_0/C:col10/1730983075278/Put/seqid=0 2024-11-07T12:37:56,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742312_1488 (size=12301) 2024-11-07T12:37:56,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/557b7482b9ce46bea6501e4b4b0cbe57 2024-11-07T12:37:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/7506e8bc41af42db90e381206b4526ae as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7506e8bc41af42db90e381206b4526ae 2024-11-07T12:37:56,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7506e8bc41af42db90e381206b4526ae, entries=150, sequenceid=384, filesize=30.5 K 2024-11-07T12:37:56,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/04d33b71059b4797be53b21d161b2f6b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/04d33b71059b4797be53b21d161b2f6b 2024-11-07T12:37:56,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/04d33b71059b4797be53b21d161b2f6b, entries=150, sequenceid=384, filesize=12.0 K 2024-11-07T12:37:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/557b7482b9ce46bea6501e4b4b0cbe57 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/557b7482b9ce46bea6501e4b4b0cbe57 2024-11-07T12:37:56,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/557b7482b9ce46bea6501e4b4b0cbe57, entries=150, sequenceid=384, filesize=12.0 K 2024-11-07T12:37:56,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=6.71 KB/6870 for 2e9c3a5f1845c325e45e01815b97b02f in 1251ms, sequenceid=384, compaction requested=false 2024-11-07T12:37:56,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2e9c3a5f1845c325e45e01815b97b02f: 2024-11-07T12:37:57,189 INFO [Thread-1697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1697. 2024-11-07T12:37:57,189 DEBUG [Thread-1697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:37:57,189 DEBUG [Thread-1697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:57,190 DEBUG [Thread-1697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:57,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1045 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3130 rows 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1050 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3144 rows 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1049 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3145 rows 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1040 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3119 rows 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1047 2024-11-07T12:37:57,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3140 rows 2024-11-07T12:37:57,190 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:37:57,190 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:37:57,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:57,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:37:57,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:37:57,191 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T12:37:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:37:57,194 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983077194"}]},"ts":"1730983077194"} 2024-11-07T12:37:57,196 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:37:57,196 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:37:57,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:37:57,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, UNASSIGN}] 2024-11-07T12:37:57,198 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, UNASSIGN 2024-11-07T12:37:57,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=2e9c3a5f1845c325e45e01815b97b02f, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:37:57,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, UNASSIGN because future has completed 2024-11-07T12:37:57,200 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:37:57,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:37:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:37:57,352 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(122): Close 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:57,352 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:37:57,352 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1722): Closing 2e9c3a5f1845c325e45e01815b97b02f, disabling compactions & flushes 2024-11-07T12:37:57,352 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:57,352 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:57,352 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. after waiting 0 ms 2024-11-07T12:37:57,352 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:57,352 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(2902): Flushing 2e9c3a5f1845c325e45e01815b97b02f 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-07T12:37:57,357 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110796f32e9c924e4550986bb69bf394a04c_2e9c3a5f1845c325e45e01815b97b02f is 50, key is test_row_1/A:col10/1730983077188/Put/seqid=0 2024-11-07T12:37:57,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742313_1489 (size=7374) 2024-11-07T12:37:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:37:57,761 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:57,764 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110796f32e9c924e4550986bb69bf394a04c_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110796f32e9c924e4550986bb69bf394a04c_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:57,765 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/31be277e986f43a7a040691e8cf9d5e5, store: [table=TestAcidGuarantees family=A region=2e9c3a5f1845c325e45e01815b97b02f] 2024-11-07T12:37:57,766 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/31be277e986f43a7a040691e8cf9d5e5 is 175, key is test_row_1/A:col10/1730983077188/Put/seqid=0 2024-11-07T12:37:57,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742314_1490 (size=13865) 2024-11-07T12:37:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:37:58,169 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/31be277e986f43a7a040691e8cf9d5e5 2024-11-07T12:37:58,175 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/b6fc9afc626a4839b9c893c1765bced0 is 50, key is test_row_1/B:col10/1730983077188/Put/seqid=0 2024-11-07T12:37:58,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742315_1491 (size=7415) 2024-11-07T12:37:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:37:58,579 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/b6fc9afc626a4839b9c893c1765bced0 2024-11-07T12:37:58,584 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/3dd98f96b34b4c4da9487d3eafaeec1e is 50, key is test_row_1/C:col10/1730983077188/Put/seqid=0 2024-11-07T12:37:58,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742316_1492 (size=7415) 2024-11-07T12:37:58,988 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/3dd98f96b34b4c4da9487d3eafaeec1e 2024-11-07T12:37:58,992 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/A/31be277e986f43a7a040691e8cf9d5e5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/31be277e986f43a7a040691e8cf9d5e5 2024-11-07T12:37:58,994 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/31be277e986f43a7a040691e8cf9d5e5, entries=50, sequenceid=389, filesize=13.5 K 2024-11-07T12:37:58,995 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/B/b6fc9afc626a4839b9c893c1765bced0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/b6fc9afc626a4839b9c893c1765bced0 2024-11-07T12:37:58,998 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/b6fc9afc626a4839b9c893c1765bced0, entries=50, sequenceid=389, filesize=7.2 K 2024-11-07T12:37:58,998 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/.tmp/C/3dd98f96b34b4c4da9487d3eafaeec1e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/3dd98f96b34b4c4da9487d3eafaeec1e 2024-11-07T12:37:59,001 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/3dd98f96b34b4c4da9487d3eafaeec1e, entries=50, sequenceid=389, filesize=7.2 K 2024-11-07T12:37:59,002 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 2e9c3a5f1845c325e45e01815b97b02f in 1650ms, sequenceid=389, compaction requested=true 2024-11-07T12:37:59,003 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9294299a30804d768b204db9caa38dc8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/24e5933542cc4c44a210cfce75310658, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36bea73ed7bd44f5830e3b307e16dc8c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/1adc08a8e13b4a12a3cb95f3773595dc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a04c768edcad4c9cbaaa6f57f19c10df, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/3a81b31a3ddc4fe8abec82026663ede4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/383aa54a63b04db7b67f6b530ce3ee8a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9860b559b531453b81543a30ee4c324d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/004ddb04aa0a4d8e9247ddbd9b72720f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/b31b52637c5549e2be0afbb26eb3b78c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/589dd62547a745e7a0f5932e385f9c38, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/30300ded9d17414fbb8646020d4024c1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7eb8ed68794d406782550fd37a09f49c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a98e2b99afe442bf8afbee87c4e3428f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/eeed016907ca4a968d50fab4728a2bd9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/82e30c6ac7a04a56ae7f9bbffc6e6937, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/33053056679b4980808ee4f82031c136, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/5a38068958a443328e722ee64dcde594, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/ea184e2763ec42b980587372af6b1cad, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/cc05555ad28a447fbcdc015dd99a31a8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/54ec44b8be264fe9b56f50dc5ce6c6e0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/6776bf4c5ce14d92a6c5ea1386290c39, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36afe3f74abc433c8bfcf2e897a568e1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/01c69e27db3a4c678308b317a9c0faf5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/20dbe3cd84814d8bbd11237567ce53ca, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/03aba7b4b8824e9eac8a7d500897da59] to archive 2024-11-07T12:37:59,003 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:59,004 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9294299a30804d768b204db9caa38dc8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9294299a30804d768b204db9caa38dc8 2024-11-07T12:37:59,005 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/24e5933542cc4c44a210cfce75310658 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/24e5933542cc4c44a210cfce75310658 2024-11-07T12:37:59,006 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36bea73ed7bd44f5830e3b307e16dc8c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36bea73ed7bd44f5830e3b307e16dc8c 2024-11-07T12:37:59,007 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/1adc08a8e13b4a12a3cb95f3773595dc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/1adc08a8e13b4a12a3cb95f3773595dc 2024-11-07T12:37:59,008 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a04c768edcad4c9cbaaa6f57f19c10df to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a04c768edcad4c9cbaaa6f57f19c10df 2024-11-07T12:37:59,008 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/3a81b31a3ddc4fe8abec82026663ede4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/3a81b31a3ddc4fe8abec82026663ede4 2024-11-07T12:37:59,009 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/383aa54a63b04db7b67f6b530ce3ee8a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/383aa54a63b04db7b67f6b530ce3ee8a 2024-11-07T12:37:59,010 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9860b559b531453b81543a30ee4c324d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/9860b559b531453b81543a30ee4c324d 2024-11-07T12:37:59,011 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/004ddb04aa0a4d8e9247ddbd9b72720f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/004ddb04aa0a4d8e9247ddbd9b72720f 2024-11-07T12:37:59,012 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/b31b52637c5549e2be0afbb26eb3b78c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/b31b52637c5549e2be0afbb26eb3b78c 2024-11-07T12:37:59,013 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/589dd62547a745e7a0f5932e385f9c38 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/589dd62547a745e7a0f5932e385f9c38 2024-11-07T12:37:59,013 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/30300ded9d17414fbb8646020d4024c1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/30300ded9d17414fbb8646020d4024c1 2024-11-07T12:37:59,014 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7eb8ed68794d406782550fd37a09f49c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7eb8ed68794d406782550fd37a09f49c 2024-11-07T12:37:59,015 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a98e2b99afe442bf8afbee87c4e3428f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/a98e2b99afe442bf8afbee87c4e3428f 2024-11-07T12:37:59,016 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/eeed016907ca4a968d50fab4728a2bd9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/eeed016907ca4a968d50fab4728a2bd9 2024-11-07T12:37:59,017 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/82e30c6ac7a04a56ae7f9bbffc6e6937 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/82e30c6ac7a04a56ae7f9bbffc6e6937 2024-11-07T12:37:59,018 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/33053056679b4980808ee4f82031c136 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/33053056679b4980808ee4f82031c136 2024-11-07T12:37:59,018 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/5a38068958a443328e722ee64dcde594 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/5a38068958a443328e722ee64dcde594 2024-11-07T12:37:59,019 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/ea184e2763ec42b980587372af6b1cad to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/ea184e2763ec42b980587372af6b1cad 2024-11-07T12:37:59,020 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/cc05555ad28a447fbcdc015dd99a31a8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/cc05555ad28a447fbcdc015dd99a31a8 2024-11-07T12:37:59,021 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/54ec44b8be264fe9b56f50dc5ce6c6e0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/54ec44b8be264fe9b56f50dc5ce6c6e0 2024-11-07T12:37:59,022 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/6776bf4c5ce14d92a6c5ea1386290c39 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/6776bf4c5ce14d92a6c5ea1386290c39 2024-11-07T12:37:59,023 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36afe3f74abc433c8bfcf2e897a568e1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/36afe3f74abc433c8bfcf2e897a568e1 2024-11-07T12:37:59,023 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/01c69e27db3a4c678308b317a9c0faf5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/01c69e27db3a4c678308b317a9c0faf5 2024-11-07T12:37:59,024 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/20dbe3cd84814d8bbd11237567ce53ca to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/20dbe3cd84814d8bbd11237567ce53ca 2024-11-07T12:37:59,025 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/03aba7b4b8824e9eac8a7d500897da59 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/03aba7b4b8824e9eac8a7d500897da59 2024-11-07T12:37:59,026 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/c9c2fa3e9c574e2eac1b37d07da625bf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/819c82eecc1d44cb870fe0f30e398eff, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/bf86d0f0e20644908e85aa2324fa1893, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/75ca896df6c840ca9061b86dac8d9b8e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/be24aebfafe74704b0a7f545c9b5cb29, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/31790e697ff74399b71965eccc4152de, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/da384bb04df646dc9383c21cb30919a6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e4274471c5e8447181ff1291ad7e9387, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b5f1e667f544623983660c277792fdc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/69b28ac5fadc4d9bb31c128408aa7b40, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/4f4b89b56b2949fcbc6fc094e7c61574, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e99104af086a4838ae826b0cfb4199bb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e757c2f0287d40be9b2d76352c9dd4c1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/544fd1879da24b63ace612966f171807, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/765a60215129433a8205ba7cdca447f4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8ec0fd94bace4ddb8d6fd28d712bb39d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/488bc81a33f14c248b84a7e624671e71, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/a621d91f64454109bf444ba90d3c5394, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/eed63dbb5d8649f890387ea094f4560b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/6a0c6aeef2304f15a599438186033ba4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b658aad9a644a41ae998e07cc2933d0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/61d7c99cef91497bb5ebde87e8fc6e11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/2a255722a8184bfa877891af956ee299, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/9624b7b9b6324cf5b06774a8b3fc252d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/1e21426479ca43f18a4d53387c47d49f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8f5ae949222546b09bc74617150aa96f] to archive 2024-11-07T12:37:59,027 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:59,028 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/c9c2fa3e9c574e2eac1b37d07da625bf to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/c9c2fa3e9c574e2eac1b37d07da625bf 2024-11-07T12:37:59,029 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/819c82eecc1d44cb870fe0f30e398eff to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/819c82eecc1d44cb870fe0f30e398eff 2024-11-07T12:37:59,030 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/bf86d0f0e20644908e85aa2324fa1893 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/bf86d0f0e20644908e85aa2324fa1893 2024-11-07T12:37:59,031 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/75ca896df6c840ca9061b86dac8d9b8e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/75ca896df6c840ca9061b86dac8d9b8e 2024-11-07T12:37:59,032 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/be24aebfafe74704b0a7f545c9b5cb29 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/be24aebfafe74704b0a7f545c9b5cb29 2024-11-07T12:37:59,032 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/31790e697ff74399b71965eccc4152de to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/31790e697ff74399b71965eccc4152de 2024-11-07T12:37:59,033 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/da384bb04df646dc9383c21cb30919a6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/da384bb04df646dc9383c21cb30919a6 2024-11-07T12:37:59,034 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e4274471c5e8447181ff1291ad7e9387 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e4274471c5e8447181ff1291ad7e9387 2024-11-07T12:37:59,035 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b5f1e667f544623983660c277792fdc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b5f1e667f544623983660c277792fdc 2024-11-07T12:37:59,036 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/69b28ac5fadc4d9bb31c128408aa7b40 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/69b28ac5fadc4d9bb31c128408aa7b40 2024-11-07T12:37:59,037 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/4f4b89b56b2949fcbc6fc094e7c61574 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/4f4b89b56b2949fcbc6fc094e7c61574 2024-11-07T12:37:59,038 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e99104af086a4838ae826b0cfb4199bb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e99104af086a4838ae826b0cfb4199bb 2024-11-07T12:37:59,038 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e757c2f0287d40be9b2d76352c9dd4c1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/e757c2f0287d40be9b2d76352c9dd4c1 2024-11-07T12:37:59,039 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/544fd1879da24b63ace612966f171807 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/544fd1879da24b63ace612966f171807 2024-11-07T12:37:59,040 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/765a60215129433a8205ba7cdca447f4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/765a60215129433a8205ba7cdca447f4 2024-11-07T12:37:59,040 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8ec0fd94bace4ddb8d6fd28d712bb39d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8ec0fd94bace4ddb8d6fd28d712bb39d 2024-11-07T12:37:59,041 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/488bc81a33f14c248b84a7e624671e71 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/488bc81a33f14c248b84a7e624671e71 2024-11-07T12:37:59,042 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/a621d91f64454109bf444ba90d3c5394 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/a621d91f64454109bf444ba90d3c5394 2024-11-07T12:37:59,043 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/eed63dbb5d8649f890387ea094f4560b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/eed63dbb5d8649f890387ea094f4560b 2024-11-07T12:37:59,043 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/6a0c6aeef2304f15a599438186033ba4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/6a0c6aeef2304f15a599438186033ba4 2024-11-07T12:37:59,044 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b658aad9a644a41ae998e07cc2933d0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/3b658aad9a644a41ae998e07cc2933d0 2024-11-07T12:37:59,045 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/61d7c99cef91497bb5ebde87e8fc6e11 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/61d7c99cef91497bb5ebde87e8fc6e11 2024-11-07T12:37:59,046 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/2a255722a8184bfa877891af956ee299 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/2a255722a8184bfa877891af956ee299 2024-11-07T12:37:59,047 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/9624b7b9b6324cf5b06774a8b3fc252d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/9624b7b9b6324cf5b06774a8b3fc252d 2024-11-07T12:37:59,047 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/1e21426479ca43f18a4d53387c47d49f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/1e21426479ca43f18a4d53387c47d49f 2024-11-07T12:37:59,048 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8f5ae949222546b09bc74617150aa96f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/8f5ae949222546b09bc74617150aa96f 2024-11-07T12:37:59,049 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/95501280adff47508edfb1a40c3aa5cf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/655073c81c0548019951a2862cde9621, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/8440b8c8ac074cef83ba7d50f94062d4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/04d9a4f901f349a6b9689bfc91d32718, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e12806e87e6c47b29adfe4925f068a90, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bff8593f651340778e0e6e04a315fd94, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/f2c9592190db4a959cef40113aef89c4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a5efcf054e77493c96b1a0dcd11b9127, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d90eb72bc0b645f88a3ec76fa3acaf89, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/b682d9c8d3b148c38261a30d99a15534, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a67fffd43f5040f1aaa6b36192463634, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/ecec3cf83d2c4fdcbfc8adb9893ad49f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/003b26b8f6154a8d8cd47129d90839fd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/6bae048907fe4d80a02a2315c9053360, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e45bcd5fa8844eb79ce5915fd47c8b7f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/0a508d734ded4c78a9584a1e0475462c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/288cf9285a8b4cbf896977e594e3c6c5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/936c13eac56a4f239c2eca123641544f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/69e7352a762a443fabdb838820820a2a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/20afbbf08f704a5082e056cbe4163d9f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/2cb1c76ff3364956b740eb2afac3d8f0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bd6827ed84724adfb0f11bb22e5c7d4d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/524c25cb7856474aa9844f958b9fd97e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/25f2c4a651e24dbe843a7776b074ecab, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d9a1812d8fbb460892fcd623bbbeb3a6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/109e8259327f4183b1b95a8524c13ecb] to archive 2024-11-07T12:37:59,050 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:37:59,051 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/95501280adff47508edfb1a40c3aa5cf to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/95501280adff47508edfb1a40c3aa5cf 2024-11-07T12:37:59,052 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/655073c81c0548019951a2862cde9621 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/655073c81c0548019951a2862cde9621 2024-11-07T12:37:59,052 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/8440b8c8ac074cef83ba7d50f94062d4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/8440b8c8ac074cef83ba7d50f94062d4 2024-11-07T12:37:59,053 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/04d9a4f901f349a6b9689bfc91d32718 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/04d9a4f901f349a6b9689bfc91d32718 2024-11-07T12:37:59,054 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e12806e87e6c47b29adfe4925f068a90 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e12806e87e6c47b29adfe4925f068a90 2024-11-07T12:37:59,055 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bff8593f651340778e0e6e04a315fd94 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bff8593f651340778e0e6e04a315fd94 2024-11-07T12:37:59,056 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/f2c9592190db4a959cef40113aef89c4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/f2c9592190db4a959cef40113aef89c4 2024-11-07T12:37:59,056 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a5efcf054e77493c96b1a0dcd11b9127 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a5efcf054e77493c96b1a0dcd11b9127 2024-11-07T12:37:59,057 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d90eb72bc0b645f88a3ec76fa3acaf89 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d90eb72bc0b645f88a3ec76fa3acaf89 2024-11-07T12:37:59,058 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/b682d9c8d3b148c38261a30d99a15534 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/b682d9c8d3b148c38261a30d99a15534 2024-11-07T12:37:59,059 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a67fffd43f5040f1aaa6b36192463634 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/a67fffd43f5040f1aaa6b36192463634 2024-11-07T12:37:59,060 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/ecec3cf83d2c4fdcbfc8adb9893ad49f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/ecec3cf83d2c4fdcbfc8adb9893ad49f 2024-11-07T12:37:59,061 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/003b26b8f6154a8d8cd47129d90839fd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/003b26b8f6154a8d8cd47129d90839fd 2024-11-07T12:37:59,061 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/6bae048907fe4d80a02a2315c9053360 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/6bae048907fe4d80a02a2315c9053360 2024-11-07T12:37:59,062 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e45bcd5fa8844eb79ce5915fd47c8b7f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/e45bcd5fa8844eb79ce5915fd47c8b7f 2024-11-07T12:37:59,063 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/0a508d734ded4c78a9584a1e0475462c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/0a508d734ded4c78a9584a1e0475462c 2024-11-07T12:37:59,063 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/288cf9285a8b4cbf896977e594e3c6c5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/288cf9285a8b4cbf896977e594e3c6c5 2024-11-07T12:37:59,064 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/936c13eac56a4f239c2eca123641544f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/936c13eac56a4f239c2eca123641544f 2024-11-07T12:37:59,065 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/69e7352a762a443fabdb838820820a2a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/69e7352a762a443fabdb838820820a2a 2024-11-07T12:37:59,066 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/20afbbf08f704a5082e056cbe4163d9f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/20afbbf08f704a5082e056cbe4163d9f 2024-11-07T12:37:59,066 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/2cb1c76ff3364956b740eb2afac3d8f0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/2cb1c76ff3364956b740eb2afac3d8f0 2024-11-07T12:37:59,067 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bd6827ed84724adfb0f11bb22e5c7d4d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/bd6827ed84724adfb0f11bb22e5c7d4d 2024-11-07T12:37:59,068 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/524c25cb7856474aa9844f958b9fd97e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/524c25cb7856474aa9844f958b9fd97e 2024-11-07T12:37:59,069 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/25f2c4a651e24dbe843a7776b074ecab to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/25f2c4a651e24dbe843a7776b074ecab 2024-11-07T12:37:59,070 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d9a1812d8fbb460892fcd623bbbeb3a6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/d9a1812d8fbb460892fcd623bbbeb3a6 2024-11-07T12:37:59,071 DEBUG [StoreCloser-TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/109e8259327f4183b1b95a8524c13ecb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/109e8259327f4183b1b95a8524c13ecb 2024-11-07T12:37:59,074 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/recovered.edits/392.seqid, newMaxSeqId=392, maxSeqId=4 2024-11-07T12:37:59,074 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. 2024-11-07T12:37:59,074 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1676): Region close journal for 2e9c3a5f1845c325e45e01815b97b02f: Waiting for close lock at 1730983077352Running coprocessor pre-close hooks at 1730983077352Disabling compacts and flushes for region at 1730983077352Disabling writes for close at 1730983077352Obtaining lock to block concurrent updates at 1730983077352Preparing flush snapshotting stores in 2e9c3a5f1845c325e45e01815b97b02f at 1730983077352Finished memstore snapshotting TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1730983077353 (+1 ms)Flushing stores of TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f. at 1730983077353Flushing 2e9c3a5f1845c325e45e01815b97b02f/A: creating writer at 1730983077353Flushing mob file 2e9c3a5f1845c325e45e01815b97b02f/A: appending metadata at 1730983077357 (+4 ms)Flushing mob file 2e9c3a5f1845c325e45e01815b97b02f/A: closing flushed file at 1730983077357Flushing 2e9c3a5f1845c325e45e01815b97b02f/A: appending metadata at 1730983077765 (+408 ms)Flushing 2e9c3a5f1845c325e45e01815b97b02f/A: closing flushed file at 1730983077765Flushing 2e9c3a5f1845c325e45e01815b97b02f/B: creating writer at 1730983078172 (+407 ms)Flushing 2e9c3a5f1845c325e45e01815b97b02f/B: appending metadata at 1730983078174 (+2 ms)Flushing 2e9c3a5f1845c325e45e01815b97b02f/B: closing flushed file at 1730983078174Flushing 2e9c3a5f1845c325e45e01815b97b02f/C: creating writer at 1730983078582 (+408 ms)Flushing 2e9c3a5f1845c325e45e01815b97b02f/C: appending metadata at 1730983078584 (+2 ms)Flushing 2e9c3a5f1845c325e45e01815b97b02f/C: closing flushed file at 1730983078584Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dd561b9: reopening flushed file at 1730983078991 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64c56563: reopening flushed file at 1730983078995 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a43042b: reopening flushed file at 1730983078998 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 2e9c3a5f1845c325e45e01815b97b02f in 1650ms, sequenceid=389, compaction requested=true at 1730983079002 (+4 ms)Writing region close event to WAL at 1730983079072 (+70 ms)Running coprocessor post-close hooks at 1730983079074 (+2 ms)Closed at 1730983079074 2024-11-07T12:37:59,076 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(157): Closed 2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,077 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=2e9c3a5f1845c325e45e01815b97b02f, regionState=CLOSED 2024-11-07T12:37:59,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:37:59,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-07T12:37:59,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseRegionProcedure 2e9c3a5f1845c325e45e01815b97b02f, server=87fb68174c52,38515,1730982961782 in 1.8790 sec 2024-11-07T12:37:59,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-07T12:37:59,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2e9c3a5f1845c325e45e01815b97b02f, UNASSIGN in 1.8830 sec 2024-11-07T12:37:59,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-07T12:37:59,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8860 sec 2024-11-07T12:37:59,085 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983079085"}]},"ts":"1730983079085"} 2024-11-07T12:37:59,086 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:37:59,086 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:37:59,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8950 sec 2024-11-07T12:37:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:37:59,337 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:59,338 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T12:37:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:59,339 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T12:37:59,340 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=125, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:59,343 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,344 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/recovered.edits] 2024-11-07T12:37:59,346 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/31be277e986f43a7a040691e8cf9d5e5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/31be277e986f43a7a040691e8cf9d5e5 2024-11-07T12:37:59,347 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/70d5d43fe4d7457dbde0edf25b39ee11 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/70d5d43fe4d7457dbde0edf25b39ee11 2024-11-07T12:37:59,348 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7506e8bc41af42db90e381206b4526ae to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/A/7506e8bc41af42db90e381206b4526ae 2024-11-07T12:37:59,350 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/04d33b71059b4797be53b21d161b2f6b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/04d33b71059b4797be53b21d161b2f6b 2024-11-07T12:37:59,351 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/7dced02d315f4af8b1b82abd301303af to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/7dced02d315f4af8b1b82abd301303af 2024-11-07T12:37:59,352 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/b6fc9afc626a4839b9c893c1765bced0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/B/b6fc9afc626a4839b9c893c1765bced0 2024-11-07T12:37:59,354 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/3dd98f96b34b4c4da9487d3eafaeec1e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/3dd98f96b34b4c4da9487d3eafaeec1e 2024-11-07T12:37:59,355 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/3ff5da3db4aa434f8733e7dc496d3f6c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/3ff5da3db4aa434f8733e7dc496d3f6c 2024-11-07T12:37:59,355 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/557b7482b9ce46bea6501e4b4b0cbe57 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/C/557b7482b9ce46bea6501e4b4b0cbe57 2024-11-07T12:37:59,357 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/recovered.edits/392.seqid to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f/recovered.edits/392.seqid 2024-11-07T12:37:59,358 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,358 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:37:59,358 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:37:59,359 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:37:59,361 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070156722360934218a346cb84947c0924_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070156722360934218a346cb84947c0924_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,362 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107103c41df2fc840fea03529e238e3d2d9_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107103c41df2fc840fea03529e238e3d2d9_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,363 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071e158e785dab49c3a18dc7494f9b4455_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071e158e785dab49c3a18dc7494f9b4455_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,364 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110722e59189abe44c1296837098627f019f_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110722e59189abe44c1296837098627f019f_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,365 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110728e448d790d9416eb3db3e97dd05959c_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110728e448d790d9416eb3db3e97dd05959c_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,365 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110728e60b982026449fad35ef152b126c5a_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110728e60b982026449fad35ef152b126c5a_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,366 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f5bea13043148cc91b6bedfce7c7c25_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f5bea13043148cc91b6bedfce7c7c25_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,367 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110746529f90e7a44508aa7cceb3ed7cab48_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110746529f90e7a44508aa7cceb3ed7cab48_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,368 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757769905b20d44cdb0fd7b62f234f5ef_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757769905b20d44cdb0fd7b62f234f5ef_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,369 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076751b4aa7d374657af3d4f1a77e15e97_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076751b4aa7d374657af3d4f1a77e15e97_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,370 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077867590b02ff42ec918938e6266ec690_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077867590b02ff42ec918938e6266ec690_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,371 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107854947e7aa974f76b0e4b4fa8f9c382d_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107854947e7aa974f76b0e4b4fa8f9c382d_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,371 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ca65759d1434087bbd3e3528cb8b3ad_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ca65759d1434087bbd3e3528cb8b3ad_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,372 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110796f32e9c924e4550986bb69bf394a04c_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110796f32e9c924e4550986bb69bf394a04c_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,373 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110798573471a9114dc39cc5bb7c5301b870_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110798573471a9114dc39cc5bb7c5301b870_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,374 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cff81b882eb0479da1f97e483ad5e596_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cff81b882eb0479da1f97e483ad5e596_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,375 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da7e064bdbac4a43a15a07ad2d61593b_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da7e064bdbac4a43a15a07ad2d61593b_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,376 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107de52cf2f14284eaa96cc72d7de158f09_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107de52cf2f14284eaa96cc72d7de158f09_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,376 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6168ba715f5487c92c7dceb7de63990_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6168ba715f5487c92c7dceb7de63990_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,377 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f19cfc360ae44f548a3a99c6af31d891_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f19cfc360ae44f548a3a99c6af31d891_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,378 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdf85afcd0ae494583178d0d8ae2cdc3_2e9c3a5f1845c325e45e01815b97b02f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdf85afcd0ae494583178d0d8ae2cdc3_2e9c3a5f1845c325e45e01815b97b02f 2024-11-07T12:37:59,379 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:37:59,381 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=125, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:59,382 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:37:59,384 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:37:59,385 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=125, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:59,385 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:37:59,385 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983079385"}]},"ts":"9223372036854775807"} 2024-11-07T12:37:59,387 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:37:59,387 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2e9c3a5f1845c325e45e01815b97b02f, NAME => 'TestAcidGuarantees,,1730983050168.2e9c3a5f1845c325e45e01815b97b02f.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:37:59,387 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:37:59,387 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983079387"}]},"ts":"9223372036854775807"} 2024-11-07T12:37:59,389 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:37:59,389 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=125, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:59,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-07T12:37:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T12:37:59,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:37:59,448 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:37:59,458 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=243 (was 242) - Thread LEAK? -, OpenFileDescriptor=465 (was 461) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 411) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6833 (was 6980) 2024-11-07T12:37:59,466 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=243, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=6832 2024-11-07T12:37:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:37:59,467 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:37:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:37:59,469 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:37:59,469 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:37:59,469 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 126 2024-11-07T12:37:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-07T12:37:59,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:37:59,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742317_1493 (size=959) 2024-11-07T12:37:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-07T12:37:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-07T12:37:59,877 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e 2024-11-07T12:37:59,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742318_1494 (size=53) 2024-11-07T12:38:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-07T12:38:00,247 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:38:00,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:38:00,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 47da5791582e51cd4fb9bc05a2daaec4, disabling compactions & flushes 2024-11-07T12:38:00,282 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:00,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:00,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. after waiting 0 ms 2024-11-07T12:38:00,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:00,282 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:00,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 47da5791582e51cd4fb9bc05a2daaec4: Waiting for close lock at 1730983080282Disabling compacts and flushes for region at 1730983080282Disabling writes for close at 1730983080282Writing region close event to WAL at 1730983080282Closed at 1730983080282 2024-11-07T12:38:00,283 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:38:00,284 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983080283"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983080283"}]},"ts":"1730983080283"} 2024-11-07T12:38:00,285 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:38:00,286 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:38:00,286 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983080286"}]},"ts":"1730983080286"} 2024-11-07T12:38:00,287 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:38:00,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, ASSIGN}] 2024-11-07T12:38:00,288 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, ASSIGN 2024-11-07T12:38:00,289 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, ASSIGN; state=OFFLINE, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=false 2024-11-07T12:38:00,440 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=47da5791582e51cd4fb9bc05a2daaec4, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:00,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, ASSIGN because future has completed 2024-11-07T12:38:00,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:38:00,596 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:00,596 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7752): Opening region: {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:38:00,597 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,597 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:38:00,597 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7794): checking encryption for 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,597 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7797): checking classloading for 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,598 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,599 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47da5791582e51cd4fb9bc05a2daaec4 columnFamilyName A 2024-11-07T12:38:00,599 DEBUG [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:00,600 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] regionserver.HStore(327): Store=47da5791582e51cd4fb9bc05a2daaec4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:00,600 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,601 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47da5791582e51cd4fb9bc05a2daaec4 columnFamilyName B 2024-11-07T12:38:00,601 DEBUG [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:00,601 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] regionserver.HStore(327): Store=47da5791582e51cd4fb9bc05a2daaec4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:00,601 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,602 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47da5791582e51cd4fb9bc05a2daaec4 columnFamilyName C 2024-11-07T12:38:00,602 DEBUG [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:00,602 INFO [StoreOpener-47da5791582e51cd4fb9bc05a2daaec4-1 {}] regionserver.HStore(327): Store=47da5791582e51cd4fb9bc05a2daaec4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:00,602 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1038): replaying wal for 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,603 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,603 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,604 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1048): stopping wal replay for 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,604 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1060): Cleaning up temporary data for 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,604 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:38:00,605 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1093): writing seq id for 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,607 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:38:00,607 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1114): Opened 47da5791582e51cd4fb9bc05a2daaec4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67605092, jitterRate=0.007394373416900635}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:38:00,607 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-07T12:38:00,608 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1006): Region open journal for 47da5791582e51cd4fb9bc05a2daaec4: Running coprocessor pre-open hook at 1730983080597Writing region info on filesystem at 1730983080597Initializing all the Stores at 1730983080598 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983080598Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983080598Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983080598Cleaning up temporary data from old regions at 1730983080604 (+6 ms)Running coprocessor post-open hooks at 1730983080607 (+3 ms)Region opened successfully at 1730983080608 (+1 ms) 2024-11-07T12:38:00,608 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., pid=128, masterSystemTime=1730983080593 2024-11-07T12:38:00,610 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:00,610 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:00,610 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=47da5791582e51cd4fb9bc05a2daaec4, regionState=OPEN, openSeqNum=2, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:00,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:38:00,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-07T12:38:00,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; OpenRegionProcedure 47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 in 170 msec 2024-11-07T12:38:00,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-07T12:38:00,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, ASSIGN in 326 msec 2024-11-07T12:38:00,615 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:38:00,616 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983080615"}]},"ts":"1730983080615"} 2024-11-07T12:38:00,617 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:38:00,617 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:38:00,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-07T12:38:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-07T12:38:01,618 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:01,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,620 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c6334f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a2ceb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,624 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:38:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb68174c52,36251,1730982961115 2024-11-07T12:38:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47786464 2024-11-07T12:38:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:38:01,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:38:01,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,627 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bcefc53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,628 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5f85fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d0bd69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,631 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6ff11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,633 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7838df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,634 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2a12e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,635 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6c89f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,636 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226ef0e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,637 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1451eecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,638 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4270faf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,640 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@718cc93c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,641 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d2d971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,642 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5500296a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eafab42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@709d28d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,646 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fecff81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,647 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25aa6d84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,648 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2643dda9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:01,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:01,650 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@386ced32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:01,651 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a151b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:01,652 DEBUG [Thread-2190 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,652 DEBUG [Thread-2205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,653 DEBUG [Thread-2194 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,653 DEBUG [Thread-2201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-07T12:38:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:38:01,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,654 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:01,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,655 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:01,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:01,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,665 DEBUG [Thread-2192 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,665 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,666 DEBUG [Thread-2209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,666 DEBUG [Thread-2196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,666 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:01,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:38:01,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,668 DEBUG [Thread-2196 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,668 DEBUG [Thread-2203 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,668 DEBUG [Thread-2196 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,669 DEBUG [Thread-2203 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,672 DEBUG [Thread-2198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:01,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:01,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2] 2024-11-07T12:38:01,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/a1e4636af4224698b9b9d6ed9c3401fe is 50, key is test_row_0/A:col10/1730983081666/Put/seqid=0 2024-11-07T12:38:01,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742319_1495 (size=12001) 2024-11-07T12:38:01,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/a1e4636af4224698b9b9d6ed9c3401fe 2024-11-07T12:38:01,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983141721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983141722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983141724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983141725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983141726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/1924a0c5d1004edcab1bbf73507a4b3e is 50, key is test_row_0/B:col10/1730983081666/Put/seqid=0 2024-11-07T12:38:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742320_1496 (size=12001) 2024-11-07T12:38:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:38:01,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:38:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:01,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:01,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983141837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983141838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983141847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983141847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983141847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:01,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:38:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:01,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:38:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983142047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983142048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983142057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983142067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983142068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:38:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/1924a0c5d1004edcab1bbf73507a4b3e 2024-11-07T12:38:02,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/a2726974b0594542b852b4a1631b17e3 is 50, key is test_row_0/C:col10/1730983081666/Put/seqid=0 2024-11-07T12:38:02,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742321_1497 (size=12001) 2024-11-07T12:38:02,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:38:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:38:02,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983142357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983142368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983142377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983142387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983142387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:38:02,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:02,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/a2726974b0594542b852b4a1631b17e3 2024-11-07T12:38:02,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:38:02,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:02,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/a1e4636af4224698b9b9d6ed9c3401fe as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/a1e4636af4224698b9b9d6ed9c3401fe 2024-11-07T12:38:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:02,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/a1e4636af4224698b9b9d6ed9c3401fe, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:38:02,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/1924a0c5d1004edcab1bbf73507a4b3e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/1924a0c5d1004edcab1bbf73507a4b3e 2024-11-07T12:38:02,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/1924a0c5d1004edcab1bbf73507a4b3e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:38:02,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/a2726974b0594542b852b4a1631b17e3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a2726974b0594542b852b4a1631b17e3 2024-11-07T12:38:02,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a2726974b0594542b852b4a1631b17e3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:38:02,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 47da5791582e51cd4fb9bc05a2daaec4 in 925ms, sequenceid=13, compaction requested=false 2024-11-07T12:38:02,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:02,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:38:02,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:02,729 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:38:02,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/4a6342f3f18e470297d2247b8141b15d is 50, key is test_row_0/A:col10/1730983081722/Put/seqid=0 2024-11-07T12:38:02,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742322_1498 (size=12001) 2024-11-07T12:38:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:38:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:02,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983142897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983142900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983142901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983142908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983142908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983143018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983143018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983143018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,142 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/4a6342f3f18e470297d2247b8141b15d 2024-11-07T12:38:03,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/bf437d2d93684fa9b29f08408a113622 is 50, key is test_row_0/B:col10/1730983081722/Put/seqid=0 2024-11-07T12:38:03,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742323_1499 (size=12001) 2024-11-07T12:38:03,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983143237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983143238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983143238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,559 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/bf437d2d93684fa9b29f08408a113622 2024-11-07T12:38:03,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983143557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983143557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983143557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/853f297596f44630bd2235f1e93c773f is 50, key is test_row_0/C:col10/1730983081722/Put/seqid=0 2024-11-07T12:38:03,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742324_1500 (size=12001) 2024-11-07T12:38:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:38:03,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983143917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,926 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983143927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:03,939 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:03,975 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/853f297596f44630bd2235f1e93c773f 2024-11-07T12:38:03,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/4a6342f3f18e470297d2247b8141b15d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4a6342f3f18e470297d2247b8141b15d 2024-11-07T12:38:03,984 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4a6342f3f18e470297d2247b8141b15d, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:38:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/bf437d2d93684fa9b29f08408a113622 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/bf437d2d93684fa9b29f08408a113622 2024-11-07T12:38:03,988 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/bf437d2d93684fa9b29f08408a113622, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:38:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/853f297596f44630bd2235f1e93c773f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/853f297596f44630bd2235f1e93c773f 2024-11-07T12:38:03,993 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/853f297596f44630bd2235f1e93c773f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:38:03,993 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 47da5791582e51cd4fb9bc05a2daaec4 in 1265ms, sequenceid=38, compaction requested=false 2024-11-07T12:38:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-07T12:38:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-07T12:38:03,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-07T12:38:03,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3400 sec 2024-11-07T12:38:04,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.3460 sec 2024-11-07T12:38:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:04,083 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:04,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:38:04,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/bb017907ef2d49cf88cab0bfd660e3c4 is 50, key is test_row_0/A:col10/1730983082895/Put/seqid=0 2024-11-07T12:38:04,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742325_1501 (size=14341) 2024-11-07T12:38:04,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983144157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983144161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983144162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,175 WARN [regionserver/87fb68174c52:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-07T12:38:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983144277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983144277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983144277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/bb017907ef2d49cf88cab0bfd660e3c4 2024-11-07T12:38:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983144497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983144497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983144497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/c505b1425aef4812a595da727b23b2a9 is 50, key is test_row_0/B:col10/1730983082895/Put/seqid=0 2024-11-07T12:38:04,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742326_1502 (size=12001) 2024-11-07T12:38:04,604 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:38:04,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983144817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983144817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983144817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:04,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/c505b1425aef4812a595da727b23b2a9 2024-11-07T12:38:04,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/8c0059a9b86348fbab1f3d0ccf6c618b is 50, key is test_row_0/C:col10/1730983082895/Put/seqid=0 2024-11-07T12:38:04,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742327_1503 (size=12001) 2024-11-07T12:38:04,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/8c0059a9b86348fbab1f3d0ccf6c618b 2024-11-07T12:38:04,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/bb017907ef2d49cf88cab0bfd660e3c4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/bb017907ef2d49cf88cab0bfd660e3c4 2024-11-07T12:38:04,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/bb017907ef2d49cf88cab0bfd660e3c4, entries=200, sequenceid=50, filesize=14.0 K 2024-11-07T12:38:04,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/c505b1425aef4812a595da727b23b2a9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c505b1425aef4812a595da727b23b2a9 2024-11-07T12:38:04,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c505b1425aef4812a595da727b23b2a9, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:38:04,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/8c0059a9b86348fbab1f3d0ccf6c618b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/8c0059a9b86348fbab1f3d0ccf6c618b 2024-11-07T12:38:04,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/8c0059a9b86348fbab1f3d0ccf6c618b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:38:04,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 47da5791582e51cd4fb9bc05a2daaec4 in 875ms, sequenceid=50, compaction requested=true 2024-11-07T12:38:04,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:04,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:04,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:04,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:04,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:04,958 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:04,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:04,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:38:04,958 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:04,959 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:04,959 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:04,959 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/A is initiating minor compaction (all files) 2024-11-07T12:38:04,959 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/B is initiating minor compaction (all files) 2024-11-07T12:38:04,959 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/B in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:04,960 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/A in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:04,960 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/1924a0c5d1004edcab1bbf73507a4b3e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/bf437d2d93684fa9b29f08408a113622, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c505b1425aef4812a595da727b23b2a9] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=35.2 K 2024-11-07T12:38:04,960 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/a1e4636af4224698b9b9d6ed9c3401fe, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4a6342f3f18e470297d2247b8141b15d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/bb017907ef2d49cf88cab0bfd660e3c4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=37.4 K 2024-11-07T12:38:04,960 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e4636af4224698b9b9d6ed9c3401fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730983081662 2024-11-07T12:38:04,960 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 1924a0c5d1004edcab1bbf73507a4b3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730983081662 2024-11-07T12:38:04,960 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a6342f3f18e470297d2247b8141b15d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730983081722 2024-11-07T12:38:04,960 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bf437d2d93684fa9b29f08408a113622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730983081722 2024-11-07T12:38:04,961 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c505b1425aef4812a595da727b23b2a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983082895 2024-11-07T12:38:04,961 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb017907ef2d49cf88cab0bfd660e3c4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983082895 2024-11-07T12:38:04,968 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:04,968 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/c93f67bc111a4520a953f82da3b6de48 is 50, key is test_row_0/B:col10/1730983082895/Put/seqid=0 2024-11-07T12:38:04,978 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#A#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:04,979 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/47500dbeb6bd4e62bfdb7c5bb50bce16 is 50, key is test_row_0/A:col10/1730983082895/Put/seqid=0 2024-11-07T12:38:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742328_1504 (size=12104) 2024-11-07T12:38:04,993 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/c93f67bc111a4520a953f82da3b6de48 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c93f67bc111a4520a953f82da3b6de48 2024-11-07T12:38:04,998 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/B of 47da5791582e51cd4fb9bc05a2daaec4 into c93f67bc111a4520a953f82da3b6de48(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:04,998 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:05,000 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/B, priority=13, startTime=1730983084958; duration=0sec 2024-11-07T12:38:05,000 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:05,000 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:B 2024-11-07T12:38:05,000 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:05,003 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:05,003 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/C is initiating minor compaction (all files) 2024-11-07T12:38:05,003 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/C in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:05,003 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a2726974b0594542b852b4a1631b17e3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/853f297596f44630bd2235f1e93c773f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/8c0059a9b86348fbab1f3d0ccf6c618b] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=35.2 K 2024-11-07T12:38:05,003 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a2726974b0594542b852b4a1631b17e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730983081662 2024-11-07T12:38:05,004 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 853f297596f44630bd2235f1e93c773f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730983081722 2024-11-07T12:38:05,004 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c0059a9b86348fbab1f3d0ccf6c618b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983082895 2024-11-07T12:38:05,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742329_1505 (size=12104) 2024-11-07T12:38:05,014 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/47500dbeb6bd4e62bfdb7c5bb50bce16 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/47500dbeb6bd4e62bfdb7c5bb50bce16 2024-11-07T12:38:05,014 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:05,015 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/ac96b9be330849bdb9b8cd87ff5c8b9c is 50, key is test_row_0/C:col10/1730983082895/Put/seqid=0 2024-11-07T12:38:05,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742330_1506 (size=12104) 2024-11-07T12:38:05,021 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/A of 47da5791582e51cd4fb9bc05a2daaec4 into 47500dbeb6bd4e62bfdb7c5bb50bce16(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:05,021 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:05,021 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/A, priority=13, startTime=1730983084958; duration=0sec 2024-11-07T12:38:05,021 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:05,021 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:A 2024-11-07T12:38:05,486 INFO [AsyncFSWAL-0-hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e-prefix:87fb68174c52,38515,1730982961782 {}] wal.AbstractFSWAL(1368): Slow sync cost: 155 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45087,DS-d4910f68-39d8-492f-9d6a-0df14bfdc14a,DISK]] 2024-11-07T12:38:05,486 INFO [AsyncFSWAL-0-hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e-prefix:87fb68174c52,38515,1730982961782 {}] wal.AbstractFSWAL(1368): Slow sync cost: 138 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45087,DS-d4910f68-39d8-492f-9d6a-0df14bfdc14a,DISK]] 2024-11-07T12:38:05,486 INFO [AsyncFSWAL-0-hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e-prefix:87fb68174c52,38515,1730982961782 {}] wal.AbstractFSWAL(1368): Slow sync cost: 137 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45087,DS-d4910f68-39d8-492f-9d6a-0df14bfdc14a,DISK]] 2024-11-07T12:38:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:05,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:05,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:38:05,491 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/ac96b9be330849bdb9b8cd87ff5c8b9c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/ac96b9be330849bdb9b8cd87ff5c8b9c 2024-11-07T12:38:05,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/fd31e6470b9e4cdb828b5e30a23d9edd is 50, key is test_row_0/A:col10/1730983084161/Put/seqid=0 2024-11-07T12:38:05,496 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/C of 47da5791582e51cd4fb9bc05a2daaec4 into ac96b9be330849bdb9b8cd87ff5c8b9c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:05,496 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:05,496 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/C, priority=13, startTime=1730983084958; duration=0sec 2024-11-07T12:38:05,496 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:05,496 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:C 2024-11-07T12:38:05,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742331_1507 (size=14341) 2024-11-07T12:38:05,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/fd31e6470b9e4cdb828b5e30a23d9edd 2024-11-07T12:38:05,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/62e880295965454ca99637b003e59ca4 is 50, key is test_row_0/B:col10/1730983084161/Put/seqid=0 2024-11-07T12:38:05,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742332_1508 (size=12001) 2024-11-07T12:38:05,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/62e880295965454ca99637b003e59ca4 2024-11-07T12:38:05,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/bdfba015330f46ca947c2a8b6e94c7ff is 50, key is test_row_0/C:col10/1730983084161/Put/seqid=0 2024-11-07T12:38:05,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983145514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983145515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983145516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742333_1509 (size=12001) 2024-11-07T12:38:05,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/bdfba015330f46ca947c2a8b6e94c7ff 2024-11-07T12:38:05,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/fd31e6470b9e4cdb828b5e30a23d9edd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fd31e6470b9e4cdb828b5e30a23d9edd 2024-11-07T12:38:05,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fd31e6470b9e4cdb828b5e30a23d9edd, entries=200, sequenceid=79, filesize=14.0 K 2024-11-07T12:38:05,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/62e880295965454ca99637b003e59ca4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/62e880295965454ca99637b003e59ca4 2024-11-07T12:38:05,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/62e880295965454ca99637b003e59ca4, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T12:38:05,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/bdfba015330f46ca947c2a8b6e94c7ff as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/bdfba015330f46ca947c2a8b6e94c7ff 2024-11-07T12:38:05,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/bdfba015330f46ca947c2a8b6e94c7ff, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T12:38:05,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 47da5791582e51cd4fb9bc05a2daaec4 in 67ms, sequenceid=79, compaction requested=false 2024-11-07T12:38:05,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:05,633 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:05,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:38:05,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/83ed3835b24544b59916722361820457 is 50, key is test_row_0/A:col10/1730983085633/Put/seqid=0 2024-11-07T12:38:05,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742334_1510 (size=11997) 2024-11-07T12:38:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983145686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983145689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983145691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:38:05,817 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:05,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:05,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-07T12:38:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983145807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983145807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983145807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,819 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:38:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,820 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:05,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:38:05,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983145937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,943 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:05,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983145967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:05,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:38:05,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:05,973 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:05,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:05,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:05,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983146028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983146028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983146028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/83ed3835b24544b59916722361820457 2024-11-07T12:38:06,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/9bff83c8e17d4757bc5964f446ba68d7 is 50, key is test_row_0/B:col10/1730983085633/Put/seqid=0 2024-11-07T12:38:06,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742335_1511 (size=9657) 2024-11-07T12:38:06,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:38:06,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:38:06,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:38:06,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:06,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983146348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983146348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983146348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:38:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:38:06,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/9bff83c8e17d4757bc5964f446ba68d7 2024-11-07T12:38:06,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7323aa031a9e4606af2c282cb0b7e7ba is 50, key is test_row_0/C:col10/1730983085633/Put/seqid=0 2024-11-07T12:38:06,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742336_1512 (size=9657) 2024-11-07T12:38:06,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:38:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:38:06,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:06,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7323aa031a9e4606af2c282cb0b7e7ba 2024-11-07T12:38:06,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983146879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983146879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983146879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:06,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/83ed3835b24544b59916722361820457 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/83ed3835b24544b59916722361820457 2024-11-07T12:38:06,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/83ed3835b24544b59916722361820457, entries=150, sequenceid=92, filesize=11.7 K 2024-11-07T12:38:06,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/9bff83c8e17d4757bc5964f446ba68d7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9bff83c8e17d4757bc5964f446ba68d7 2024-11-07T12:38:06,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9bff83c8e17d4757bc5964f446ba68d7, entries=100, sequenceid=92, filesize=9.4 K 2024-11-07T12:38:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7323aa031a9e4606af2c282cb0b7e7ba as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7323aa031a9e4606af2c282cb0b7e7ba 2024-11-07T12:38:06,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:38:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:06,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:06,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7323aa031a9e4606af2c282cb0b7e7ba, entries=100, sequenceid=92, filesize=9.4 K 2024-11-07T12:38:06,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 47da5791582e51cd4fb9bc05a2daaec4 in 1268ms, sequenceid=92, compaction requested=true 2024-11-07T12:38:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:06,901 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:06,901 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:06,902 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:06,902 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:06,902 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/A is initiating minor compaction (all files) 2024-11-07T12:38:06,902 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/B is initiating minor compaction (all files) 2024-11-07T12:38:06,902 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/A in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,902 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/B in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,902 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/47500dbeb6bd4e62bfdb7c5bb50bce16, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fd31e6470b9e4cdb828b5e30a23d9edd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/83ed3835b24544b59916722361820457] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=37.5 K 2024-11-07T12:38:06,902 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c93f67bc111a4520a953f82da3b6de48, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/62e880295965454ca99637b003e59ca4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9bff83c8e17d4757bc5964f446ba68d7] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=33.0 K 2024-11-07T12:38:06,903 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c93f67bc111a4520a953f82da3b6de48, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983082895 2024-11-07T12:38:06,903 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47500dbeb6bd4e62bfdb7c5bb50bce16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983082895 2024-11-07T12:38:06,903 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e880295965454ca99637b003e59ca4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730983084161 2024-11-07T12:38:06,903 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd31e6470b9e4cdb828b5e30a23d9edd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730983084158 2024-11-07T12:38:06,903 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bff83c8e17d4757bc5964f446ba68d7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730983085498 2024-11-07T12:38:06,903 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83ed3835b24544b59916722361820457, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730983085498 2024-11-07T12:38:06,912 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:06,913 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/78466839c4214d769e93a4aca3920723 is 50, key is test_row_0/B:col10/1730983085633/Put/seqid=0 2024-11-07T12:38:06,913 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#A#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:06,913 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/6876bdad98cf45f8b80312747fa619b6 is 50, key is test_row_0/A:col10/1730983085633/Put/seqid=0 2024-11-07T12:38:06,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742338_1514 (size=12207) 2024-11-07T12:38:06,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742337_1513 (size=12207) 2024-11-07T12:38:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:38:06,959 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/78466839c4214d769e93a4aca3920723 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/78466839c4214d769e93a4aca3920723 2024-11-07T12:38:06,964 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/6876bdad98cf45f8b80312747fa619b6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6876bdad98cf45f8b80312747fa619b6 2024-11-07T12:38:06,969 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/B of 47da5791582e51cd4fb9bc05a2daaec4 into 78466839c4214d769e93a4aca3920723(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:06,969 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:06,969 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/B, priority=13, startTime=1730983086901; duration=0sec 2024-11-07T12:38:06,969 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:06,969 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:B 2024-11-07T12:38:06,969 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:06,971 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:06,971 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/C is initiating minor compaction (all files) 2024-11-07T12:38:06,971 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/C in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:06,971 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/ac96b9be330849bdb9b8cd87ff5c8b9c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/bdfba015330f46ca947c2a8b6e94c7ff, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7323aa031a9e4606af2c282cb0b7e7ba] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=33.0 K 2024-11-07T12:38:06,972 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting ac96b9be330849bdb9b8cd87ff5c8b9c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983082895 2024-11-07T12:38:06,972 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfba015330f46ca947c2a8b6e94c7ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730983084161 2024-11-07T12:38:06,972 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/A of 47da5791582e51cd4fb9bc05a2daaec4 into 6876bdad98cf45f8b80312747fa619b6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:06,972 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:06,972 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/A, priority=13, startTime=1730983086901; duration=0sec 2024-11-07T12:38:06,972 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:06,972 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:A 2024-11-07T12:38:06,974 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 7323aa031a9e4606af2c282cb0b7e7ba, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730983085498 2024-11-07T12:38:06,981 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:06,981 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/a7b3344fedbf423097d00088d0e9a59a is 50, key is test_row_0/C:col10/1730983085633/Put/seqid=0 2024-11-07T12:38:07,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742339_1515 (size=12207) 2024-11-07T12:38:07,014 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/a7b3344fedbf423097d00088d0e9a59a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a7b3344fedbf423097d00088d0e9a59a 2024-11-07T12:38:07,019 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/C of 47da5791582e51cd4fb9bc05a2daaec4 into a7b3344fedbf423097d00088d0e9a59a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:07,022 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:07,022 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/C, priority=13, startTime=1730983086901; duration=0sec 2024-11-07T12:38:07,022 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:07,022 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:C 2024-11-07T12:38:07,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:38:07,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:07,052 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:38:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/6aa52d78a9934c2283610cc43b9c0ba8 is 50, key is test_row_0/A:col10/1730983085688/Put/seqid=0 2024-11-07T12:38:07,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742340_1516 (size=12001) 2024-11-07T12:38:07,477 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/6aa52d78a9934c2283610cc43b9c0ba8 2024-11-07T12:38:07,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/eddf70ff6bea45b388bfb42c8ca5ccb8 is 50, key is test_row_0/B:col10/1730983085688/Put/seqid=0 2024-11-07T12:38:07,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742341_1517 (size=12001) 2024-11-07T12:38:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:07,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:07,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983147913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:07,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983147916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:07,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:07,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:07,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:07,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983147917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:07,939 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/eddf70ff6bea45b388bfb42c8ca5ccb8 2024-11-07T12:38:07,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7de96ea856b74b639efff1d9c220a09b is 50, key is test_row_0/C:col10/1730983085688/Put/seqid=0 2024-11-07T12:38:07,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742342_1518 (size=12001) 2024-11-07T12:38:07,959 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7de96ea856b74b639efff1d9c220a09b 2024-11-07T12:38:07,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/6aa52d78a9934c2283610cc43b9c0ba8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6aa52d78a9934c2283610cc43b9c0ba8 2024-11-07T12:38:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:38:07,969 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6aa52d78a9934c2283610cc43b9c0ba8, entries=150, sequenceid=119, filesize=11.7 K 2024-11-07T12:38:07,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/eddf70ff6bea45b388bfb42c8ca5ccb8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/eddf70ff6bea45b388bfb42c8ca5ccb8 2024-11-07T12:38:07,980 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/eddf70ff6bea45b388bfb42c8ca5ccb8, entries=150, sequenceid=119, filesize=11.7 K 2024-11-07T12:38:07,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7de96ea856b74b639efff1d9c220a09b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7de96ea856b74b639efff1d9c220a09b 2024-11-07T12:38:07,984 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7de96ea856b74b639efff1d9c220a09b, entries=150, sequenceid=119, filesize=11.7 K 2024-11-07T12:38:07,985 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 47da5791582e51cd4fb9bc05a2daaec4 in 933ms, sequenceid=119, compaction requested=false 2024-11-07T12:38:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-07T12:38:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-07T12:38:07,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-07T12:38:07,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1690 sec 2024-11-07T12:38:07,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.1750 sec 2024-11-07T12:38:08,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:08,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:38:08,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b03e08b393014ac7b749c836edd22cf0 is 50, key is test_row_0/A:col10/1730983087903/Put/seqid=0 2024-11-07T12:38:08,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742343_1519 (size=14491) 2024-11-07T12:38:08,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b03e08b393014ac7b749c836edd22cf0 2024-11-07T12:38:08,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/8179e9a9fda643ff9faf31af68581674 is 50, key is test_row_0/B:col10/1730983087903/Put/seqid=0 2024-11-07T12:38:08,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742344_1520 (size=12101) 2024-11-07T12:38:08,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/8179e9a9fda643ff9faf31af68581674 2024-11-07T12:38:08,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/75f7bbe9f1874c37a24b032aa7cc9d9d is 50, key is test_row_0/C:col10/1730983087903/Put/seqid=0 2024-11-07T12:38:08,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742345_1521 (size=12101) 2024-11-07T12:38:08,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/75f7bbe9f1874c37a24b032aa7cc9d9d 2024-11-07T12:38:08,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b03e08b393014ac7b749c836edd22cf0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b03e08b393014ac7b749c836edd22cf0 2024-11-07T12:38:08,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b03e08b393014ac7b749c836edd22cf0, entries=200, sequenceid=132, filesize=14.2 K 2024-11-07T12:38:08,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/8179e9a9fda643ff9faf31af68581674 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8179e9a9fda643ff9faf31af68581674 2024-11-07T12:38:08,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8179e9a9fda643ff9faf31af68581674, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T12:38:08,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/75f7bbe9f1874c37a24b032aa7cc9d9d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75f7bbe9f1874c37a24b032aa7cc9d9d 2024-11-07T12:38:08,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75f7bbe9f1874c37a24b032aa7cc9d9d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T12:38:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983148095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47da5791582e51cd4fb9bc05a2daaec4 in 77ms, sequenceid=132, compaction requested=true 2024-11-07T12:38:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:08,109 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:08,109 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,111 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:08,111 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/B is initiating minor compaction (all files) 2024-11-07T12:38:08,111 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/B in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:08,111 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/78466839c4214d769e93a4aca3920723, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/eddf70ff6bea45b388bfb42c8ca5ccb8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8179e9a9fda643ff9faf31af68581674] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=35.5 K 2024-11-07T12:38:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:08,111 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:08,111 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 78466839c4214d769e93a4aca3920723, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730983085347 2024-11-07T12:38:08,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:38:08,111 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:08,111 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/A is initiating minor compaction (all files) 2024-11-07T12:38:08,111 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/A in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:08,112 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting eddf70ff6bea45b388bfb42c8ca5ccb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730983085684 2024-11-07T12:38:08,112 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6876bdad98cf45f8b80312747fa619b6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6aa52d78a9934c2283610cc43b9c0ba8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b03e08b393014ac7b749c836edd22cf0] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=37.8 K 2024-11-07T12:38:08,112 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6876bdad98cf45f8b80312747fa619b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730983085347 2024-11-07T12:38:08,112 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8179e9a9fda643ff9faf31af68581674, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983087903 2024-11-07T12:38:08,113 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aa52d78a9934c2283610cc43b9c0ba8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730983085684 2024-11-07T12:38:08,113 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b03e08b393014ac7b749c836edd22cf0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983087903 2024-11-07T12:38:08,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/c12880cdc8b347b498af030021c64b06 is 50, key is test_row_0/A:col10/1730983088099/Put/seqid=0 2024-11-07T12:38:08,121 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#B#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:08,121 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/051d31aac1b240cbbaba66f867f59c95 is 50, key is test_row_0/B:col10/1730983087903/Put/seqid=0 2024-11-07T12:38:08,124 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:08,124 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/89d68ca1cab5422a967e653da886f6bc is 50, key is test_row_0/A:col10/1730983087903/Put/seqid=0 2024-11-07T12:38:08,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742346_1522 (size=14541) 2024-11-07T12:38:08,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742347_1523 (size=12409) 2024-11-07T12:38:08,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742348_1524 (size=12409) 2024-11-07T12:38:08,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983148148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,154 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/89d68ca1cab5422a967e653da886f6bc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/89d68ca1cab5422a967e653da886f6bc 2024-11-07T12:38:08,160 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/A of 47da5791582e51cd4fb9bc05a2daaec4 into 89d68ca1cab5422a967e653da886f6bc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:08,160 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:08,160 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/A, priority=13, startTime=1730983088109; duration=0sec 2024-11-07T12:38:08,160 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:08,160 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:A 2024-11-07T12:38:08,160 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:08,161 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:08,161 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/C is initiating minor compaction (all files) 2024-11-07T12:38:08,161 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/C in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:08,161 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a7b3344fedbf423097d00088d0e9a59a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7de96ea856b74b639efff1d9c220a09b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75f7bbe9f1874c37a24b032aa7cc9d9d] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=35.5 K 2024-11-07T12:38:08,161 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7b3344fedbf423097d00088d0e9a59a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730983085347 2024-11-07T12:38:08,162 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de96ea856b74b639efff1d9c220a09b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730983085684 2024-11-07T12:38:08,162 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f7bbe9f1874c37a24b032aa7cc9d9d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983087903 2024-11-07T12:38:08,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983148153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,168 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:08,169 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/267749ca711a412dbfa11af34f35c613 is 50, key is test_row_0/C:col10/1730983087903/Put/seqid=0 2024-11-07T12:38:08,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742349_1525 (size=12409) 2024-11-07T12:38:08,176 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/267749ca711a412dbfa11af34f35c613 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/267749ca711a412dbfa11af34f35c613 2024-11-07T12:38:08,181 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/C of 47da5791582e51cd4fb9bc05a2daaec4 into 267749ca711a412dbfa11af34f35c613(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:08,181 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:08,181 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/C, priority=13, startTime=1730983088109; duration=0sec 2024-11-07T12:38:08,181 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:08,181 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:C 2024-11-07T12:38:08,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983148217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983148257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983148267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983148438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983148467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983148487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/c12880cdc8b347b498af030021c64b06 2024-11-07T12:38:08,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/0a6bff407ddd4c36b4d8246b9737f848 is 50, key is test_row_0/B:col10/1730983088099/Put/seqid=0 2024-11-07T12:38:08,551 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/051d31aac1b240cbbaba66f867f59c95 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/051d31aac1b240cbbaba66f867f59c95 2024-11-07T12:38:08,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742350_1526 (size=12151) 2024-11-07T12:38:08,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/0a6bff407ddd4c36b4d8246b9737f848 2024-11-07T12:38:08,556 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/B of 47da5791582e51cd4fb9bc05a2daaec4 into 051d31aac1b240cbbaba66f867f59c95(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:08,556 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:08,556 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/B, priority=13, startTime=1730983088109; duration=0sec 2024-11-07T12:38:08,556 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:08,556 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:B 2024-11-07T12:38:08,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/f6e0797d54f24ac5a91330d2f5fe444a is 50, key is test_row_0/C:col10/1730983088099/Put/seqid=0 2024-11-07T12:38:08,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742351_1527 (size=12151) 2024-11-07T12:38:08,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/f6e0797d54f24ac5a91330d2f5fe444a 2024-11-07T12:38:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/c12880cdc8b347b498af030021c64b06 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c12880cdc8b347b498af030021c64b06 2024-11-07T12:38:08,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c12880cdc8b347b498af030021c64b06, entries=200, sequenceid=156, filesize=14.2 K 2024-11-07T12:38:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-07T12:38:08,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/0a6bff407ddd4c36b4d8246b9737f848 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/0a6bff407ddd4c36b4d8246b9737f848 2024-11-07T12:38:08,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/0a6bff407ddd4c36b4d8246b9737f848, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:38:08,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/f6e0797d54f24ac5a91330d2f5fe444a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f6e0797d54f24ac5a91330d2f5fe444a 2024-11-07T12:38:08,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f6e0797d54f24ac5a91330d2f5fe444a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:38:08,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 47da5791582e51cd4fb9bc05a2daaec4 in 479ms, sequenceid=156, compaction requested=false 2024-11-07T12:38:08,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:08,762 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:08,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:38:08,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/1714d81242a54aa2814d162f83d52174 is 50, key is test_row_0/A:col10/1730983088151/Put/seqid=0 2024-11-07T12:38:08,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742352_1528 (size=14541) 2024-11-07T12:38:08,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983148845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983148854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983148857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983148967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983148978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983148978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:09,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/1714d81242a54aa2814d162f83d52174 2024-11-07T12:38:09,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/8e4a91de6cc64fb1b96394fc2c3b17a2 is 50, key is test_row_0/B:col10/1730983088151/Put/seqid=0 2024-11-07T12:38:09,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983149177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:09,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742353_1529 (size=12151) 2024-11-07T12:38:09,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/8e4a91de6cc64fb1b96394fc2c3b17a2 2024-11-07T12:38:09,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/1c73e58bafbd42e18813ba4ff27126fe is 50, key is test_row_0/C:col10/1730983088151/Put/seqid=0 2024-11-07T12:38:09,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983149204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:09,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983149205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:09,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742354_1530 (size=12151) 2024-11-07T12:38:09,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983149488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:09,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:09,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983149517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:09,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983149527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:09,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:09,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:09,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/1c73e58bafbd42e18813ba4ff27126fe 2024-11-07T12:38:09,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/1714d81242a54aa2814d162f83d52174 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/1714d81242a54aa2814d162f83d52174 2024-11-07T12:38:09,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/1714d81242a54aa2814d162f83d52174, entries=200, sequenceid=172, filesize=14.2 K 2024-11-07T12:38:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/8e4a91de6cc64fb1b96394fc2c3b17a2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e4a91de6cc64fb1b96394fc2c3b17a2 2024-11-07T12:38:09,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e4a91de6cc64fb1b96394fc2c3b17a2, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T12:38:09,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/1c73e58bafbd42e18813ba4ff27126fe as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/1c73e58bafbd42e18813ba4ff27126fe 2024-11-07T12:38:09,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/1c73e58bafbd42e18813ba4ff27126fe, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T12:38:09,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47da5791582e51cd4fb9bc05a2daaec4 in 874ms, sequenceid=172, compaction requested=true 2024-11-07T12:38:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:09,637 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:09,637 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:09,638 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:09,638 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:09,638 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/A is initiating minor compaction (all files) 2024-11-07T12:38:09,638 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/B is initiating minor compaction (all files) 2024-11-07T12:38:09,638 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/A in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:09,638 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/B in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:09,638 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/89d68ca1cab5422a967e653da886f6bc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c12880cdc8b347b498af030021c64b06, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/1714d81242a54aa2814d162f83d52174] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=40.5 K 2024-11-07T12:38:09,638 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/051d31aac1b240cbbaba66f867f59c95, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/0a6bff407ddd4c36b4d8246b9737f848, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e4a91de6cc64fb1b96394fc2c3b17a2] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=35.9 K 2024-11-07T12:38:09,638 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d68ca1cab5422a967e653da886f6bc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983087903 2024-11-07T12:38:09,638 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 051d31aac1b240cbbaba66f867f59c95, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983087903 2024-11-07T12:38:09,638 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c12880cdc8b347b498af030021c64b06, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730983088082 2024-11-07T12:38:09,638 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6bff407ddd4c36b4d8246b9737f848, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730983088092 2024-11-07T12:38:09,639 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1714d81242a54aa2814d162f83d52174, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983088128 2024-11-07T12:38:09,639 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e4a91de6cc64fb1b96394fc2c3b17a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983088128 2024-11-07T12:38:09,645 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:09,645 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b1141b9ea2e94235be0f8066d3aa4eaf is 50, key is test_row_0/A:col10/1730983088151/Put/seqid=0 2024-11-07T12:38:09,648 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:09,648 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/7f28ea9bd6684d2caca4f54c8db261cf is 50, key is test_row_0/B:col10/1730983088151/Put/seqid=0 2024-11-07T12:38:09,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742355_1531 (size=12561) 2024-11-07T12:38:09,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742356_1532 (size=12561) 2024-11-07T12:38:09,662 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/7f28ea9bd6684d2caca4f54c8db261cf as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/7f28ea9bd6684d2caca4f54c8db261cf 2024-11-07T12:38:09,668 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/B of 47da5791582e51cd4fb9bc05a2daaec4 into 7f28ea9bd6684d2caca4f54c8db261cf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:09,668 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:09,668 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/B, priority=13, startTime=1730983089637; duration=0sec 2024-11-07T12:38:09,668 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:09,668 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:B 2024-11-07T12:38:09,668 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:09,669 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:09,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/C is initiating minor compaction (all files) 2024-11-07T12:38:09,670 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/C in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:09,670 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/267749ca711a412dbfa11af34f35c613, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f6e0797d54f24ac5a91330d2f5fe444a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/1c73e58bafbd42e18813ba4ff27126fe] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=35.9 K 2024-11-07T12:38:09,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 267749ca711a412dbfa11af34f35c613, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730983087903 2024-11-07T12:38:09,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f6e0797d54f24ac5a91330d2f5fe444a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730983088092 2024-11-07T12:38:09,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c73e58bafbd42e18813ba4ff27126fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983088128 2024-11-07T12:38:09,679 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:09,680 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/c657ad4554cf4f48a696318e6784b692 is 50, key is test_row_0/C:col10/1730983088151/Put/seqid=0 2024-11-07T12:38:09,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742357_1533 (size=12561) 2024-11-07T12:38:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:09,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:38:09,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/0886f485c4554fa1b7edfc2e8c827922 is 50, key is test_row_0/A:col10/1730983088824/Put/seqid=0 2024-11-07T12:38:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:38:09,987 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:09,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-07T12:38:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:38:09,995 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:09,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742358_1534 (size=14541) 2024-11-07T12:38:09,996 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:09,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:09,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/0886f485c4554fa1b7edfc2e8c827922 2024-11-07T12:38:10,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/5bf50fd49d2043dbbe63841530eff037 is 50, key is test_row_0/B:col10/1730983088824/Put/seqid=0 2024-11-07T12:38:10,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742359_1535 (size=12151) 2024-11-07T12:38:10,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983150030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983150035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983150035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983150037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983150048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,063 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b1141b9ea2e94235be0f8066d3aa4eaf as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b1141b9ea2e94235be0f8066d3aa4eaf 2024-11-07T12:38:10,067 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/A of 47da5791582e51cd4fb9bc05a2daaec4 into b1141b9ea2e94235be0f8066d3aa4eaf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:10,067 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:10,067 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/A, priority=13, startTime=1730983089637; duration=0sec 2024-11-07T12:38:10,067 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:10,067 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:A 2024-11-07T12:38:10,092 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/c657ad4554cf4f48a696318e6784b692 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c657ad4554cf4f48a696318e6784b692 2024-11-07T12:38:10,096 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/C of 47da5791582e51cd4fb9bc05a2daaec4 into c657ad4554cf4f48a696318e6784b692(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:10,096 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:10,096 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/C, priority=13, startTime=1730983089637; duration=0sec 2024-11-07T12:38:10,096 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:10,096 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:C 2024-11-07T12:38:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:38:10,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:38:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:10,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:10,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983150148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983150148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983150148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:38:10,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:10,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:10,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:10,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:10,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:38:10,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983150368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983150368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983150368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/5bf50fd49d2043dbbe63841530eff037 2024-11-07T12:38:10,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/30571bd459944d0db3d0bcc86863fcb4 is 50, key is test_row_0/C:col10/1730983088824/Put/seqid=0 2024-11-07T12:38:10,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742360_1536 (size=12151) 2024-11-07T12:38:10,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/30571bd459944d0db3d0bcc86863fcb4 2024-11-07T12:38:10,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/0886f485c4554fa1b7edfc2e8c827922 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/0886f485c4554fa1b7edfc2e8c827922 2024-11-07T12:38:10,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/0886f485c4554fa1b7edfc2e8c827922, entries=200, sequenceid=197, filesize=14.2 K 2024-11-07T12:38:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/5bf50fd49d2043dbbe63841530eff037 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/5bf50fd49d2043dbbe63841530eff037 2024-11-07T12:38:10,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/5bf50fd49d2043dbbe63841530eff037, entries=150, sequenceid=197, filesize=11.9 K 2024-11-07T12:38:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/30571bd459944d0db3d0bcc86863fcb4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/30571bd459944d0db3d0bcc86863fcb4 2024-11-07T12:38:10,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/30571bd459944d0db3d0bcc86863fcb4, entries=150, sequenceid=197, filesize=11.9 K 2024-11-07T12:38:10,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 47da5791582e51cd4fb9bc05a2daaec4 in 464ms, sequenceid=197, compaction requested=false 2024-11-07T12:38:10,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:10,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:38:10,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:10,458 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:38:10,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/3c656f101f754512aed00aad412c7d40 is 50, key is test_row_0/A:col10/1730983090017/Put/seqid=0 2024-11-07T12:38:10,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742361_1537 (size=12151) 2024-11-07T12:38:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:38:10,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:10,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983150754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983150755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983150760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,867 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/3c656f101f754512aed00aad412c7d40 2024-11-07T12:38:10,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/302bcdcb3374480da8a0a704809d4451 is 50, key is test_row_0/B:col10/1730983090017/Put/seqid=0 2024-11-07T12:38:10,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983150877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983150877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983150877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:10,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742362_1538 (size=12151) 2024-11-07T12:38:11,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983151057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,067 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:11,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983151068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,079 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983151103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983151103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983151103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:38:11,290 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/302bcdcb3374480da8a0a704809d4451 2024-11-07T12:38:11,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/32df2d6455754b409ce4b8d010f03d4e is 50, key is test_row_0/C:col10/1730983090017/Put/seqid=0 2024-11-07T12:38:11,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742363_1539 (size=12151) 2024-11-07T12:38:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983151417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983151417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983151417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:11,702 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/32df2d6455754b409ce4b8d010f03d4e 2024-11-07T12:38:11,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/3c656f101f754512aed00aad412c7d40 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/3c656f101f754512aed00aad412c7d40 2024-11-07T12:38:11,709 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/3c656f101f754512aed00aad412c7d40, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T12:38:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/302bcdcb3374480da8a0a704809d4451 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/302bcdcb3374480da8a0a704809d4451 2024-11-07T12:38:11,713 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/302bcdcb3374480da8a0a704809d4451, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T12:38:11,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/32df2d6455754b409ce4b8d010f03d4e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/32df2d6455754b409ce4b8d010f03d4e 2024-11-07T12:38:11,717 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/32df2d6455754b409ce4b8d010f03d4e, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T12:38:11,718 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47da5791582e51cd4fb9bc05a2daaec4 in 1260ms, sequenceid=212, compaction requested=true 2024-11-07T12:38:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-07T12:38:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-07T12:38:11,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-07T12:38:11,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7230 sec 2024-11-07T12:38:11,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7340 sec 2024-11-07T12:38:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:11,945 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:11,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:38:11,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/591e04385b4647a799095114e9dfd1b4 is 50, key is test_row_0/A:col10/1730983091943/Put/seqid=0 2024-11-07T12:38:11,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742364_1540 (size=14541) 2024-11-07T12:38:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983151976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983151990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983151991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983152100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983152118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983152118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:38:12,137 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:12,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-07T12:38:12,139 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:38:12,140 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:12,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:38:12,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:38:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:12,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:12,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983152318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983152337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983152339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/591e04385b4647a799095114e9dfd1b4 2024-11-07T12:38:12,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/721e44cd452f4640b3c9e2dec3aae427 is 50, key is test_row_0/B:col10/1730983091943/Put/seqid=0 2024-11-07T12:38:12,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742365_1541 (size=12151) 2024-11-07T12:38:12,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/721e44cd452f4640b3c9e2dec3aae427 2024-11-07T12:38:12,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/4b7d53e6cb5545b48ae298642ab1521e is 50, key is test_row_0/C:col10/1730983091943/Put/seqid=0 2024-11-07T12:38:12,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:38:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:12,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:12,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:38:12,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742366_1542 (size=12151) 2024-11-07T12:38:12,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/4b7d53e6cb5545b48ae298642ab1521e 2024-11-07T12:38:12,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/591e04385b4647a799095114e9dfd1b4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/591e04385b4647a799095114e9dfd1b4 2024-11-07T12:38:12,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/591e04385b4647a799095114e9dfd1b4, entries=200, sequenceid=236, filesize=14.2 K 2024-11-07T12:38:12,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/721e44cd452f4640b3c9e2dec3aae427 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/721e44cd452f4640b3c9e2dec3aae427 2024-11-07T12:38:12,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/721e44cd452f4640b3c9e2dec3aae427, entries=150, sequenceid=236, filesize=11.9 K 2024-11-07T12:38:12,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/4b7d53e6cb5545b48ae298642ab1521e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/4b7d53e6cb5545b48ae298642ab1521e 2024-11-07T12:38:12,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/4b7d53e6cb5545b48ae298642ab1521e, entries=150, sequenceid=236, filesize=11.9 K 2024-11-07T12:38:12,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 47da5791582e51cd4fb9bc05a2daaec4 in 571ms, sequenceid=236, compaction requested=true 2024-11-07T12:38:12,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:12,517 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:12,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:12,518 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:12,518 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/A is initiating minor compaction (all files) 2024-11-07T12:38:12,518 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/A in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:12,518 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b1141b9ea2e94235be0f8066d3aa4eaf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/0886f485c4554fa1b7edfc2e8c827922, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/3c656f101f754512aed00aad412c7d40, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/591e04385b4647a799095114e9dfd1b4] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=52.5 K 2024-11-07T12:38:12,518 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1141b9ea2e94235be0f8066d3aa4eaf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983088128 2024-11-07T12:38:12,519 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0886f485c4554fa1b7edfc2e8c827922, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730983088824 2024-11-07T12:38:12,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:12,519 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c656f101f754512aed00aad412c7d40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983090017 2024-11-07T12:38:12,519 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:12,520 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 591e04385b4647a799095114e9dfd1b4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730983090752 2024-11-07T12:38:12,521 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:12,521 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/B is initiating minor compaction (all files) 2024-11-07T12:38:12,521 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/B in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:12,522 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/7f28ea9bd6684d2caca4f54c8db261cf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/5bf50fd49d2043dbbe63841530eff037, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/302bcdcb3374480da8a0a704809d4451, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/721e44cd452f4640b3c9e2dec3aae427] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=47.9 K 2024-11-07T12:38:12,522 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f28ea9bd6684d2caca4f54c8db261cf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983088128 2024-11-07T12:38:12,522 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bf50fd49d2043dbbe63841530eff037, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730983088824 2024-11-07T12:38:12,523 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 302bcdcb3374480da8a0a704809d4451, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983090017 2024-11-07T12:38:12,524 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 721e44cd452f4640b3c9e2dec3aae427, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730983090752 2024-11-07T12:38:12,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:12,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:12,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:12,543 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#A#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:12,544 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/521adac2eafc4de199aaa3a1607d6051 is 50, key is test_row_0/A:col10/1730983091943/Put/seqid=0 2024-11-07T12:38:12,558 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#B#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:12,559 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/75af55c80c004b9f8aeb16e9813740c2 is 50, key is test_row_0/B:col10/1730983091943/Put/seqid=0 2024-11-07T12:38:12,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742367_1543 (size=12697) 2024-11-07T12:38:12,590 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/521adac2eafc4de199aaa3a1607d6051 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/521adac2eafc4de199aaa3a1607d6051 2024-11-07T12:38:12,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742368_1544 (size=12697) 2024-11-07T12:38:12,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:38:12,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:12,601 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:38:12,603 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/A of 47da5791582e51cd4fb9bc05a2daaec4 into 521adac2eafc4de199aaa3a1607d6051(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:12,604 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:12,604 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/A, priority=12, startTime=1730983092516; duration=0sec 2024-11-07T12:38:12,604 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:12,604 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:A 2024-11-07T12:38:12,604 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:12,612 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:12,612 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/C is initiating minor compaction (all files) 2024-11-07T12:38:12,612 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/C in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:12,612 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c657ad4554cf4f48a696318e6784b692, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/30571bd459944d0db3d0bcc86863fcb4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/32df2d6455754b409ce4b8d010f03d4e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/4b7d53e6cb5545b48ae298642ab1521e] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=47.9 K 2024-11-07T12:38:12,612 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c657ad4554cf4f48a696318e6784b692, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983088128 2024-11-07T12:38:12,613 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30571bd459944d0db3d0bcc86863fcb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730983088824 2024-11-07T12:38:12,613 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32df2d6455754b409ce4b8d010f03d4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983090017 2024-11-07T12:38:12,613 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b7d53e6cb5545b48ae298642ab1521e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730983090752 2024-11-07T12:38:12,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/5a82398a218b474ebb745a31ea787868 is 50, key is test_row_0/A:col10/1730983091988/Put/seqid=0 2024-11-07T12:38:12,638 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#C#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:12,638 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/73b6e8c7831144388cfdf90784f3d640 is 50, key is test_row_0/C:col10/1730983091943/Put/seqid=0 2024-11-07T12:38:12,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:12,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742369_1545 (size=12151) 2024-11-07T12:38:12,650 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/5a82398a218b474ebb745a31ea787868 2024-11-07T12:38:12,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742370_1546 (size=12697) 2024-11-07T12:38:12,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/138645212c514b34848d31752b4253f4 is 50, key is test_row_0/B:col10/1730983091988/Put/seqid=0 2024-11-07T12:38:12,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742371_1547 (size=12151) 2024-11-07T12:38:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:38:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983152780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983152783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983152784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983152897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983152897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:12,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983152909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,005 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/75af55c80c004b9f8aeb16e9813740c2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/75af55c80c004b9f8aeb16e9813740c2 2024-11-07T12:38:13,015 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/B of 47da5791582e51cd4fb9bc05a2daaec4 into 75af55c80c004b9f8aeb16e9813740c2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:13,015 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:13,015 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/B, priority=12, startTime=1730983092519; duration=0sec 2024-11-07T12:38:13,015 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:13,015 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:B 2024-11-07T12:38:13,091 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/73b6e8c7831144388cfdf90784f3d640 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/73b6e8c7831144388cfdf90784f3d640 2024-11-07T12:38:13,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983153087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983153087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,093 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:13,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,093 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,102 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/C of 47da5791582e51cd4fb9bc05a2daaec4 into 73b6e8c7831144388cfdf90784f3d640(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:13,103 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:13,103 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/C, priority=12, startTime=1730983092524; duration=0sec 2024-11-07T12:38:13,103 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:13,103 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:C 2024-11-07T12:38:13,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983153118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983153118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983153129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,147 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/138645212c514b34848d31752b4253f4 2024-11-07T12:38:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/07c234dae52b4d5192bfee67d6ef11bd is 50, key is test_row_0/C:col10/1730983091988/Put/seqid=0 2024-11-07T12:38:13,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742372_1548 (size=12151) 2024-11-07T12:38:13,187 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/07c234dae52b4d5192bfee67d6ef11bd 2024-11-07T12:38:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/5a82398a218b474ebb745a31ea787868 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5a82398a218b474ebb745a31ea787868 2024-11-07T12:38:13,201 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5a82398a218b474ebb745a31ea787868, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T12:38:13,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/138645212c514b34848d31752b4253f4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/138645212c514b34848d31752b4253f4 2024-11-07T12:38:13,208 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/138645212c514b34848d31752b4253f4, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T12:38:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/07c234dae52b4d5192bfee67d6ef11bd as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/07c234dae52b4d5192bfee67d6ef11bd 2024-11-07T12:38:13,217 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/07c234dae52b4d5192bfee67d6ef11bd, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T12:38:13,218 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47da5791582e51cd4fb9bc05a2daaec4 in 617ms, sequenceid=249, compaction requested=false 2024-11-07T12:38:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-07T12:38:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-07T12:38:13,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-07T12:38:13,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0820 sec 2024-11-07T12:38:13,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0900 sec 2024-11-07T12:38:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:38:13,278 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:13,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-07T12:38:13,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:38:13,282 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:13,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:38:13,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-07T12:38:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:13,435 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:38:13,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b49bc13134ae45a58d3039ef0d368d62 is 50, key is test_row_0/A:col10/1730983092762/Put/seqid=0 2024-11-07T12:38:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:13,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:13,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742373_1549 (size=12301) 2024-11-07T12:38:13,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983153474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983153476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983153479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:38:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983153597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983153598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983153598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983153807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983153808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983153817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:13,862 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b49bc13134ae45a58d3039ef0d368d62 2024-11-07T12:38:13,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/e2f11acd5fc749278db47aef6ae14398 is 50, key is test_row_0/B:col10/1730983092762/Put/seqid=0 2024-11-07T12:38:13,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742374_1550 (size=12301) 2024-11-07T12:38:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:38:14,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983154127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983154128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983154138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,292 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/e2f11acd5fc749278db47aef6ae14398 2024-11-07T12:38:14,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/eac0b8cdbcdd43c29310ab26a9d47369 is 50, key is test_row_0/C:col10/1730983092762/Put/seqid=0 2024-11-07T12:38:14,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742375_1551 (size=12301) 2024-11-07T12:38:14,303 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/eac0b8cdbcdd43c29310ab26a9d47369 2024-11-07T12:38:14,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b49bc13134ae45a58d3039ef0d368d62 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b49bc13134ae45a58d3039ef0d368d62 2024-11-07T12:38:14,310 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b49bc13134ae45a58d3039ef0d368d62, entries=150, sequenceid=275, filesize=12.0 K 2024-11-07T12:38:14,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/e2f11acd5fc749278db47aef6ae14398 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/e2f11acd5fc749278db47aef6ae14398 2024-11-07T12:38:14,314 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/e2f11acd5fc749278db47aef6ae14398, entries=150, sequenceid=275, filesize=12.0 K 2024-11-07T12:38:14,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/eac0b8cdbcdd43c29310ab26a9d47369 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/eac0b8cdbcdd43c29310ab26a9d47369 2024-11-07T12:38:14,323 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/eac0b8cdbcdd43c29310ab26a9d47369, entries=150, sequenceid=275, filesize=12.0 K 2024-11-07T12:38:14,324 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 47da5791582e51cd4fb9bc05a2daaec4 in 889ms, sequenceid=275, compaction requested=true 2024-11-07T12:38:14,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:14,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:14,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-07T12:38:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-07T12:38:14,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-07T12:38:14,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0470 sec 2024-11-07T12:38:14,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.0550 sec 2024-11-07T12:38:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:38:14,419 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:14,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-07T12:38:14,422 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:38:14,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:14,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:38:14,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T12:38:14,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:14,576 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:38:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b5a1c1ee5c0749ff96485e87a334fd1d is 50, key is test_row_0/A:col10/1730983093460/Put/seqid=0 2024-11-07T12:38:14,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742376_1552 (size=12301) 2024-11-07T12:38:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:14,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:14,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983154718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983154720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983154721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:38:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983154837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983154838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983154841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:14,991 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b5a1c1ee5c0749ff96485e87a334fd1d 2024-11-07T12:38:14,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/61809c92f7c54ad89d1752e9fb7ca75d is 50, key is test_row_0/B:col10/1730983093460/Put/seqid=0 2024-11-07T12:38:15,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742377_1553 (size=12301) 2024-11-07T12:38:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:38:15,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983155057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983155057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983155057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983155378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:15,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983155378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983155378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:15,404 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/61809c92f7c54ad89d1752e9fb7ca75d 2024-11-07T12:38:15,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/5b43bed3eb224af1ba02945cf936e249 is 50, key is test_row_0/C:col10/1730983093460/Put/seqid=0 2024-11-07T12:38:15,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742378_1554 (size=12301) 2024-11-07T12:38:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:38:15,834 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/5b43bed3eb224af1ba02945cf936e249 2024-11-07T12:38:15,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b5a1c1ee5c0749ff96485e87a334fd1d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b5a1c1ee5c0749ff96485e87a334fd1d 2024-11-07T12:38:15,842 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b5a1c1ee5c0749ff96485e87a334fd1d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T12:38:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/61809c92f7c54ad89d1752e9fb7ca75d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/61809c92f7c54ad89d1752e9fb7ca75d 2024-11-07T12:38:15,846 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/61809c92f7c54ad89d1752e9fb7ca75d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T12:38:15,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/5b43bed3eb224af1ba02945cf936e249 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5b43bed3eb224af1ba02945cf936e249 2024-11-07T12:38:15,851 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5b43bed3eb224af1ba02945cf936e249, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T12:38:15,852 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47da5791582e51cd4fb9bc05a2daaec4 in 1276ms, sequenceid=288, compaction requested=true 2024-11-07T12:38:15,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:15,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:15,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-07T12:38:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-07T12:38:15,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-07T12:38:15,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-07T12:38:15,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.4350 sec 2024-11-07T12:38:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:15,905 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:15,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:38:15,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/7388b63baff74483815234190008c030 is 50, key is test_row_0/A:col10/1730983094717/Put/seqid=0 2024-11-07T12:38:15,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742379_1555 (size=14741) 2024-11-07T12:38:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983155935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:15,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983155936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983155938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983156047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983156057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983156057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983156267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983156278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983156278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/7388b63baff74483815234190008c030 2024-11-07T12:38:16,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/90a813cfbc94416982428655f3980608 is 50, key is test_row_0/B:col10/1730983094717/Put/seqid=0 2024-11-07T12:38:16,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742380_1556 (size=12301) 2024-11-07T12:38:16,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/90a813cfbc94416982428655f3980608 2024-11-07T12:38:16,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/b506603f59614ee6b645982f6a0801cf is 50, key is test_row_0/C:col10/1730983094717/Put/seqid=0 2024-11-07T12:38:16,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742381_1557 (size=12301) 2024-11-07T12:38:16,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/b506603f59614ee6b645982f6a0801cf 2024-11-07T12:38:16,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/7388b63baff74483815234190008c030 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/7388b63baff74483815234190008c030 2024-11-07T12:38:16,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/7388b63baff74483815234190008c030, entries=200, sequenceid=312, filesize=14.4 K 2024-11-07T12:38:16,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/90a813cfbc94416982428655f3980608 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/90a813cfbc94416982428655f3980608 2024-11-07T12:38:16,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/90a813cfbc94416982428655f3980608, entries=150, sequenceid=312, filesize=12.0 K 2024-11-07T12:38:16,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/b506603f59614ee6b645982f6a0801cf as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/b506603f59614ee6b645982f6a0801cf 2024-11-07T12:38:16,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/b506603f59614ee6b645982f6a0801cf, entries=150, sequenceid=312, filesize=12.0 K 2024-11-07T12:38:16,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 47da5791582e51cd4fb9bc05a2daaec4 in 593ms, sequenceid=312, compaction requested=true 2024-11-07T12:38:16,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:16,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:16,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:16,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:16,500 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:38:16,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:16,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:16,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:16,500 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:38:16,503 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:38:16,503 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/B is initiating minor compaction (all files) 2024-11-07T12:38:16,503 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/B in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:16,504 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/75af55c80c004b9f8aeb16e9813740c2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/138645212c514b34848d31752b4253f4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/e2f11acd5fc749278db47aef6ae14398, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/61809c92f7c54ad89d1752e9fb7ca75d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/90a813cfbc94416982428655f3980608] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=60.3 K 2024-11-07T12:38:16,504 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64191 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:38:16,504 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/A is initiating minor compaction (all files) 2024-11-07T12:38:16,504 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/A in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:16,504 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/521adac2eafc4de199aaa3a1607d6051, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5a82398a218b474ebb745a31ea787868, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b49bc13134ae45a58d3039ef0d368d62, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b5a1c1ee5c0749ff96485e87a334fd1d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/7388b63baff74483815234190008c030] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=62.7 K 2024-11-07T12:38:16,504 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 75af55c80c004b9f8aeb16e9813740c2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730983090752 2024-11-07T12:38:16,504 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521adac2eafc4de199aaa3a1607d6051, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730983090752 2024-11-07T12:38:16,504 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 138645212c514b34848d31752b4253f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730983091970 2024-11-07T12:38:16,505 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f11acd5fc749278db47aef6ae14398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730983092741 2024-11-07T12:38:16,505 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a82398a218b474ebb745a31ea787868, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730983091970 2024-11-07T12:38:16,505 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 61809c92f7c54ad89d1752e9fb7ca75d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730983093460 2024-11-07T12:38:16,506 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b49bc13134ae45a58d3039ef0d368d62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730983092741 2024-11-07T12:38:16,506 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a813cfbc94416982428655f3980608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730983094717 2024-11-07T12:38:16,506 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a1c1ee5c0749ff96485e87a334fd1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730983093460 2024-11-07T12:38:16,506 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7388b63baff74483815234190008c030, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730983094700 2024-11-07T12:38:16,559 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#A#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:16,559 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/c2e500cd0882405bb397dbdc2c5df957 is 50, key is test_row_0/A:col10/1730983094717/Put/seqid=0 2024-11-07T12:38:16,567 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:16,567 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/9c42c287079f4cc88754673ff9555df8 is 50, key is test_row_0/B:col10/1730983094717/Put/seqid=0 2024-11-07T12:38:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:38:16,569 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:16,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-07T12:38:16,571 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:38:16,572 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:16,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:16,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742382_1558 (size=13017) 2024-11-07T12:38:16,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742383_1559 (size=13017) 2024-11-07T12:38:16,594 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:16,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:38:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:16,596 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/9c42c287079f4cc88754673ff9555df8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c42c287079f4cc88754673ff9555df8 2024-11-07T12:38:16,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/41b74780d8d747c1a8dde23b9bfb0647 is 50, key is test_row_0/A:col10/1730983095935/Put/seqid=0 2024-11-07T12:38:16,605 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/B of 47da5791582e51cd4fb9bc05a2daaec4 into 9c42c287079f4cc88754673ff9555df8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:16,605 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:16,605 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/B, priority=11, startTime=1730983096500; duration=0sec 2024-11-07T12:38:16,606 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:16,606 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:B 2024-11-07T12:38:16,606 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:38:16,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742384_1560 (size=14741) 2024-11-07T12:38:16,609 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:38:16,611 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/C is initiating minor compaction (all files) 2024-11-07T12:38:16,611 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/C in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:16,611 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/73b6e8c7831144388cfdf90784f3d640, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/07c234dae52b4d5192bfee67d6ef11bd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/eac0b8cdbcdd43c29310ab26a9d47369, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5b43bed3eb224af1ba02945cf936e249, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/b506603f59614ee6b645982f6a0801cf] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=60.3 K 2024-11-07T12:38:16,611 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b6e8c7831144388cfdf90784f3d640, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730983090752 2024-11-07T12:38:16,611 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c234dae52b4d5192bfee67d6ef11bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730983091970 2024-11-07T12:38:16,612 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting eac0b8cdbcdd43c29310ab26a9d47369, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730983092741 2024-11-07T12:38:16,612 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b43bed3eb224af1ba02945cf936e249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730983093460 2024-11-07T12:38:16,613 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting b506603f59614ee6b645982f6a0801cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730983094717 2024-11-07T12:38:16,635 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:16,636 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/9452aa7fb24b458e8c8d3be035b62dd6 is 50, key is test_row_0/C:col10/1730983094717/Put/seqid=0 2024-11-07T12:38:16,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742385_1561 (size=13017) 2024-11-07T12:38:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:38:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983156704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983156705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:38:16,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983156711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:16,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:16,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983156841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983156841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983156841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:16,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:38:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:16,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:38:16,991 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/c2e500cd0882405bb397dbdc2c5df957 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c2e500cd0882405bb397dbdc2c5df957 2024-11-07T12:38:16,996 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/A of 47da5791582e51cd4fb9bc05a2daaec4 into c2e500cd0882405bb397dbdc2c5df957(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:16,996 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:16,996 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/A, priority=11, startTime=1730983096500; duration=0sec 2024-11-07T12:38:16,996 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:16,996 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:A 2024-11-07T12:38:17,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/41b74780d8d747c1a8dde23b9bfb0647 2024-11-07T12:38:17,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/791e6809e4824df1a60cbd8105c02bac is 50, key is test_row_0/B:col10/1730983095935/Put/seqid=0 2024-11-07T12:38:17,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:38:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:17,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742386_1562 (size=12301) 2024-11-07T12:38:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/791e6809e4824df1a60cbd8105c02bac 2024-11-07T12:38:17,055 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/9452aa7fb24b458e8c8d3be035b62dd6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/9452aa7fb24b458e8c8d3be035b62dd6 2024-11-07T12:38:17,062 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/C of 47da5791582e51cd4fb9bc05a2daaec4 into 9452aa7fb24b458e8c8d3be035b62dd6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:17,062 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:17,062 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/C, priority=11, startTime=1730983096500; duration=0sec 2024-11-07T12:38:17,062 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:17,062 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:C 2024-11-07T12:38:17,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/76a9ab0eb8c24727bdf8edebd313488a is 50, key is test_row_0/C:col10/1730983095935/Put/seqid=0 2024-11-07T12:38:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983157058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983157058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983157058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742387_1563 (size=12301) 2024-11-07T12:38:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56548 deadline: 1730983157137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56600 deadline: 1730983157137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,143 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,144 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_1' in 47da5791582e51cd4fb9bc05a2daaec4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8290 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:38:17,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:38:17,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:17,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:38:17,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:17,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:17,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:17,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:17,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983157377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983157378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983157378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/76a9ab0eb8c24727bdf8edebd313488a 2024-11-07T12:38:17,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/41b74780d8d747c1a8dde23b9bfb0647 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/41b74780d8d747c1a8dde23b9bfb0647 2024-11-07T12:38:17,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/41b74780d8d747c1a8dde23b9bfb0647, entries=200, sequenceid=325, filesize=14.4 K 2024-11-07T12:38:17,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/791e6809e4824df1a60cbd8105c02bac as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/791e6809e4824df1a60cbd8105c02bac 2024-11-07T12:38:17,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/791e6809e4824df1a60cbd8105c02bac, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T12:38:17,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/76a9ab0eb8c24727bdf8edebd313488a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/76a9ab0eb8c24727bdf8edebd313488a 2024-11-07T12:38:17,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/76a9ab0eb8c24727bdf8edebd313488a, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T12:38:17,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47da5791582e51cd4fb9bc05a2daaec4 in 919ms, sequenceid=325, compaction requested=false 2024-11-07T12:38:17,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:17,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:38:17,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:17,520 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:38:17,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/4d0bebd8d5054af69202125eb1309756 is 50, key is test_row_0/A:col10/1730983096702/Put/seqid=0 2024-11-07T12:38:17,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742388_1564 (size=12301) 2024-11-07T12:38:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:38:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983157935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983157935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983157936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:17,984 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/4d0bebd8d5054af69202125eb1309756 2024-11-07T12:38:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/628a3de3120d4a7ba4c6899f0456438c is 50, key is test_row_0/B:col10/1730983096702/Put/seqid=0 2024-11-07T12:38:18,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742389_1565 (size=12301) 2024-11-07T12:38:18,012 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/628a3de3120d4a7ba4c6899f0456438c 2024-11-07T12:38:18,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/a197f052744e46b3b920bf9a1d7a4ed0 is 50, key is test_row_0/C:col10/1730983096702/Put/seqid=0 2024-11-07T12:38:18,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742390_1566 (size=12301) 2024-11-07T12:38:18,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983158057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983158057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983158057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983158269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983158273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983158277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,433 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/a197f052744e46b3b920bf9a1d7a4ed0 2024-11-07T12:38:18,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/4d0bebd8d5054af69202125eb1309756 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4d0bebd8d5054af69202125eb1309756 2024-11-07T12:38:18,529 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4d0bebd8d5054af69202125eb1309756, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T12:38:18,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/628a3de3120d4a7ba4c6899f0456438c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/628a3de3120d4a7ba4c6899f0456438c 2024-11-07T12:38:18,556 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/628a3de3120d4a7ba4c6899f0456438c, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T12:38:18,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/a197f052744e46b3b920bf9a1d7a4ed0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a197f052744e46b3b920bf9a1d7a4ed0 2024-11-07T12:38:18,593 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a197f052744e46b3b920bf9a1d7a4ed0, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T12:38:18,594 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 47da5791582e51cd4fb9bc05a2daaec4 in 1074ms, sequenceid=351, compaction requested=true 2024-11-07T12:38:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-07T12:38:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-07T12:38:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:18,602 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:18,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:38:18,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-07T12:38:18,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0290 sec 2024-11-07T12:38:18,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/002eb2ed2f364ad08bf074d7e3d1aae8 is 50, key is test_row_0/A:col10/1730983098599/Put/seqid=0 2024-11-07T12:38:18,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 2.0400 sec 2024-11-07T12:38:18,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742391_1567 (size=14741) 2024-11-07T12:38:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:38:18,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,727 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983158711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983158711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983158714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-07T12:38:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:38:18,731 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:18,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:18,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:38:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983158837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983158838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983158838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:18,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T12:38:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:18,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T12:38:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:38:19,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/002eb2ed2f364ad08bf074d7e3d1aae8 2024-11-07T12:38:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983159048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983159048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983159048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:19,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/a299a79d962549d7aa33ccb1f1a8ea93 is 50, key is test_row_0/B:col10/1730983098599/Put/seqid=0 2024-11-07T12:38:19,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742392_1568 (size=12301) 2024-11-07T12:38:19,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T12:38:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T12:38:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:38:19,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983159368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983159368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983159368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:19,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T12:38:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:19,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/a299a79d962549d7aa33ccb1f1a8ea93 2024-11-07T12:38:19,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/c0690a73566645ec8800cf241f30353f is 50, key is test_row_0/C:col10/1730983098599/Put/seqid=0 2024-11-07T12:38:19,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742393_1569 (size=12301) 2024-11-07T12:38:19,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/c0690a73566645ec8800cf241f30353f 2024-11-07T12:38:19,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/002eb2ed2f364ad08bf074d7e3d1aae8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/002eb2ed2f364ad08bf074d7e3d1aae8 2024-11-07T12:38:19,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/002eb2ed2f364ad08bf074d7e3d1aae8, entries=200, sequenceid=367, filesize=14.4 K 2024-11-07T12:38:19,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/a299a79d962549d7aa33ccb1f1a8ea93 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/a299a79d962549d7aa33ccb1f1a8ea93 2024-11-07T12:38:19,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/a299a79d962549d7aa33ccb1f1a8ea93, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T12:38:19,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/c0690a73566645ec8800cf241f30353f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c0690a73566645ec8800cf241f30353f 2024-11-07T12:38:19,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c0690a73566645ec8800cf241f30353f, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T12:38:19,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 47da5791582e51cd4fb9bc05a2daaec4 in 1063ms, sequenceid=367, compaction requested=true 2024-11-07T12:38:19,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:19,665 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:19,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:19,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:19,666 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:19,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:19,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:19,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:19,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:19,666 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:19,669 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/A is initiating minor compaction (all files) 2024-11-07T12:38:19,669 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/A in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,669 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c2e500cd0882405bb397dbdc2c5df957, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/41b74780d8d747c1a8dde23b9bfb0647, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4d0bebd8d5054af69202125eb1309756, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/002eb2ed2f364ad08bf074d7e3d1aae8] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=53.5 K 2024-11-07T12:38:19,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:19,670 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e500cd0882405bb397dbdc2c5df957, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730983094717 2024-11-07T12:38:19,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T12:38:19,675 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/B is initiating minor compaction (all files) 2024-11-07T12:38:19,675 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/B in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,675 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c42c287079f4cc88754673ff9555df8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/791e6809e4824df1a60cbd8105c02bac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/628a3de3120d4a7ba4c6899f0456438c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/a299a79d962549d7aa33ccb1f1a8ea93] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=48.8 K 2024-11-07T12:38:19,675 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b74780d8d747c1a8dde23b9bfb0647, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730983095927 2024-11-07T12:38:19,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:19,676 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T12:38:19,677 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c42c287079f4cc88754673ff9555df8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730983094717 2024-11-07T12:38:19,677 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 791e6809e4824df1a60cbd8105c02bac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730983095927 2024-11-07T12:38:19,677 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d0bebd8d5054af69202125eb1309756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730983096690 2024-11-07T12:38:19,680 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 628a3de3120d4a7ba4c6899f0456438c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730983096690 2024-11-07T12:38:19,680 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 002eb2ed2f364ad08bf074d7e3d1aae8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983097918 2024-11-07T12:38:19,681 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a299a79d962549d7aa33ccb1f1a8ea93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983097918 2024-11-07T12:38:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/5f882e859b784a91899b291fdbe6a39d is 50, key is test_row_0/A:col10/1730983098711/Put/seqid=0 2024-11-07T12:38:19,694 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#B#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:19,695 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/8e428e9be57b4290a81996c2d6ccdc5d is 50, key is test_row_0/B:col10/1730983098599/Put/seqid=0 2024-11-07T12:38:19,706 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#A#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:19,706 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/fc20854eddbd413c8be86294366de49e is 50, key is test_row_0/A:col10/1730983098599/Put/seqid=0 2024-11-07T12:38:19,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742394_1570 (size=12301) 2024-11-07T12:38:19,725 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/5f882e859b784a91899b291fdbe6a39d 2024-11-07T12:38:19,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742395_1571 (size=13153) 2024-11-07T12:38:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/9c44ba4cd23940e2a53d7b98451ed598 is 50, key is test_row_0/B:col10/1730983098711/Put/seqid=0 2024-11-07T12:38:19,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742396_1572 (size=13153) 2024-11-07T12:38:19,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742397_1573 (size=12301) 2024-11-07T12:38:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:38:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:19,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:19,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983159940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983159942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:19,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983159943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983160067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983160067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983160067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,151 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/fc20854eddbd413c8be86294366de49e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fc20854eddbd413c8be86294366de49e 2024-11-07T12:38:20,152 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/8e428e9be57b4290a81996c2d6ccdc5d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e428e9be57b4290a81996c2d6ccdc5d 2024-11-07T12:38:20,157 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/A of 47da5791582e51cd4fb9bc05a2daaec4 into fc20854eddbd413c8be86294366de49e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:20,157 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:20,157 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/A, priority=12, startTime=1730983099665; duration=0sec 2024-11-07T12:38:20,157 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:20,157 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:A 2024-11-07T12:38:20,157 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:20,158 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/B of 47da5791582e51cd4fb9bc05a2daaec4 into 8e428e9be57b4290a81996c2d6ccdc5d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:20,158 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:20,158 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/B, priority=12, startTime=1730983099666; duration=0sec 2024-11-07T12:38:20,158 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:20,158 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:B 2024-11-07T12:38:20,159 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:20,159 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/C is initiating minor compaction (all files) 2024-11-07T12:38:20,159 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/C in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:20,159 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/9452aa7fb24b458e8c8d3be035b62dd6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/76a9ab0eb8c24727bdf8edebd313488a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a197f052744e46b3b920bf9a1d7a4ed0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c0690a73566645ec8800cf241f30353f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=48.8 K 2024-11-07T12:38:20,160 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9452aa7fb24b458e8c8d3be035b62dd6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730983094717 2024-11-07T12:38:20,160 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76a9ab0eb8c24727bdf8edebd313488a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730983095927 2024-11-07T12:38:20,161 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting a197f052744e46b3b920bf9a1d7a4ed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730983096690 2024-11-07T12:38:20,161 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0690a73566645ec8800cf241f30353f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983097918 2024-11-07T12:38:20,169 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:20,169 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/fc630d4f98ba470da0662310bede18a6 is 50, key is test_row_0/C:col10/1730983098599/Put/seqid=0 2024-11-07T12:38:20,170 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/9c44ba4cd23940e2a53d7b98451ed598 2024-11-07T12:38:20,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742398_1574 (size=13153) 2024-11-07T12:38:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/f10220d88d0c4187b1a5e9917732e97f is 50, key is test_row_0/C:col10/1730983098711/Put/seqid=0 2024-11-07T12:38:20,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742399_1575 (size=12301) 2024-11-07T12:38:20,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983160287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983160288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983160288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,583 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/fc630d4f98ba470da0662310bede18a6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/fc630d4f98ba470da0662310bede18a6 2024-11-07T12:38:20,587 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/C of 47da5791582e51cd4fb9bc05a2daaec4 into fc630d4f98ba470da0662310bede18a6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:20,587 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:20,587 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/C, priority=12, startTime=1730983099666; duration=0sec 2024-11-07T12:38:20,588 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:20,588 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:C 2024-11-07T12:38:20,599 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/f10220d88d0c4187b1a5e9917732e97f 2024-11-07T12:38:20,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/5f882e859b784a91899b291fdbe6a39d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5f882e859b784a91899b291fdbe6a39d 2024-11-07T12:38:20,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983160598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983160607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983160607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:20,622 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5f882e859b784a91899b291fdbe6a39d, entries=150, sequenceid=387, filesize=12.0 K 2024-11-07T12:38:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/9c44ba4cd23940e2a53d7b98451ed598 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c44ba4cd23940e2a53d7b98451ed598 2024-11-07T12:38:20,629 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c44ba4cd23940e2a53d7b98451ed598, entries=150, sequenceid=387, filesize=12.0 K 2024-11-07T12:38:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/f10220d88d0c4187b1a5e9917732e97f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f10220d88d0c4187b1a5e9917732e97f 2024-11-07T12:38:20,637 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f10220d88d0c4187b1a5e9917732e97f, entries=150, sequenceid=387, filesize=12.0 K 2024-11-07T12:38:20,638 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 47da5791582e51cd4fb9bc05a2daaec4 in 962ms, sequenceid=387, compaction requested=false 2024-11-07T12:38:20,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:20,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:20,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-07T12:38:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-07T12:38:20,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-07T12:38:20,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9120 sec 2024-11-07T12:38:20,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.9200 sec 2024-11-07T12:38:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:38:20,877 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:20,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-07T12:38:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:38:20,880 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:20,881 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:20,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:38:21,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-07T12:38:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:21,033 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T12:38:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/851b81ed35f54ffb8fed8d5a851870c6 is 50, key is test_row_0/A:col10/1730983099941/Put/seqid=0 2024-11-07T12:38:21,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742400_1576 (size=12301) 2024-11-07T12:38:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:21,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. as already flushing 2024-11-07T12:38:21,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983161175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983161177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:38:21,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983161186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983161298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983161298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983161307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,442 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/851b81ed35f54ffb8fed8d5a851870c6 2024-11-07T12:38:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/495538a6618d4a2eb470e7b2756964f6 is 50, key is test_row_0/B:col10/1730983099941/Put/seqid=0 2024-11-07T12:38:21,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742401_1577 (size=12301) 2024-11-07T12:38:21,455 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/495538a6618d4a2eb470e7b2756964f6 2024-11-07T12:38:21,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/75b65bec563a4169b935eaa31ad949fa is 50, key is test_row_0/C:col10/1730983099941/Put/seqid=0 2024-11-07T12:38:21,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742402_1578 (size=12301) 2024-11-07T12:38:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:38:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983161508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983161508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983161528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,669 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-11-07T12:38:21,669 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:21,669 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,670 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:21,674 INFO [Thread-2201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2201. 2024-11-07T12:38:21,674 DEBUG [Thread-2201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:21,674 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,674 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,674 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:21,676 INFO [Thread-2205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2205. 2024-11-07T12:38:21,676 DEBUG [Thread-2205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:21,676 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,676 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:21,681 INFO [Thread-2203 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2203. 2024-11-07T12:38:21,681 DEBUG [Thread-2203 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:21,681 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,681 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:21,684 INFO [Thread-2209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2209. 2024-11-07T12:38:21,684 DEBUG [Thread-2209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:21,684 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,684 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:21,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:21,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56532 deadline: 1730983161827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56572 deadline: 1730983161827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56616 deadline: 1730983161837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., hostname=87fb68174c52,38515,1730982961782, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:38:21,879 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/75b65bec563a4169b935eaa31ad949fa 2024-11-07T12:38:21,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/851b81ed35f54ffb8fed8d5a851870c6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/851b81ed35f54ffb8fed8d5a851870c6 2024-11-07T12:38:21,886 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/851b81ed35f54ffb8fed8d5a851870c6, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:38:21,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/495538a6618d4a2eb470e7b2756964f6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/495538a6618d4a2eb470e7b2756964f6 2024-11-07T12:38:21,889 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/495538a6618d4a2eb470e7b2756964f6, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:38:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/75b65bec563a4169b935eaa31ad949fa as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75b65bec563a4169b935eaa31ad949fa 2024-11-07T12:38:21,892 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75b65bec563a4169b935eaa31ad949fa, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:38:21,893 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 47da5791582e51cd4fb9bc05a2daaec4 in 860ms, sequenceid=406, compaction requested=true 2024-11-07T12:38:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-07T12:38:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-07T12:38:21,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-07T12:38:21,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-11-07T12:38:21,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.0190 sec 2024-11-07T12:38:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:38:22,018 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:22,339 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:22,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:38:22,339 INFO [Thread-2192 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2192. 2024-11-07T12:38:22,339 INFO [Thread-2190 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2190. 2024-11-07T12:38:22,339 DEBUG [Thread-2190 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:22,339 DEBUG [Thread-2192 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:22,339 DEBUG [Thread-2190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:22,339 DEBUG [Thread-2192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:22,339 DEBUG [Thread-2190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:22,339 DEBUG [Thread-2192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:22,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:22,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:22,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/2488ec52cc5e4dba81a012f927bf9f8c is 50, key is test_row_0/A:col10/1730983101175/Put/seqid=0 2024-11-07T12:38:22,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742403_1579 (size=12301) 2024-11-07T12:38:22,349 INFO [Thread-2198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2198. 2024-11-07T12:38:22,349 DEBUG [Thread-2198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:22,349 DEBUG [Thread-2198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:22,349 DEBUG [Thread-2198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:22,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:22,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/2488ec52cc5e4dba81a012f927bf9f8c 2024-11-07T12:38:22,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/4ce6ea0457d84537b1deddf94dc7129c is 50, key is test_row_0/B:col10/1730983101175/Put/seqid=0 2024-11-07T12:38:22,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742404_1580 (size=12301) 2024-11-07T12:38:23,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/4ce6ea0457d84537b1deddf94dc7129c 2024-11-07T12:38:23,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/aca443223eb047b9ac63a789bd2e1b36 is 50, key is test_row_0/C:col10/1730983101175/Put/seqid=0 2024-11-07T12:38:23,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742405_1581 (size=12301) 2024-11-07T12:38:23,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/aca443223eb047b9ac63a789bd2e1b36 2024-11-07T12:38:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/2488ec52cc5e4dba81a012f927bf9f8c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/2488ec52cc5e4dba81a012f927bf9f8c 2024-11-07T12:38:23,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/2488ec52cc5e4dba81a012f927bf9f8c, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:38:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/4ce6ea0457d84537b1deddf94dc7129c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/4ce6ea0457d84537b1deddf94dc7129c 2024-11-07T12:38:23,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/4ce6ea0457d84537b1deddf94dc7129c, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:38:23,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/aca443223eb047b9ac63a789bd2e1b36 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/aca443223eb047b9ac63a789bd2e1b36 2024-11-07T12:38:23,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/aca443223eb047b9ac63a789bd2e1b36, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:38:23,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=6.71 KB/6870 for 47da5791582e51cd4fb9bc05a2daaec4 in 1243ms, sequenceid=428, compaction requested=true 2024-11-07T12:38:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:23,583 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47da5791582e51cd4fb9bc05a2daaec4:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:23,583 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:23,584 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:23,584 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:23,584 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/B is initiating minor compaction (all files) 2024-11-07T12:38:23,584 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/A is initiating minor compaction (all files) 2024-11-07T12:38:23,584 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/B in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:23,584 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/A in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:23,584 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e428e9be57b4290a81996c2d6ccdc5d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c44ba4cd23940e2a53d7b98451ed598, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/495538a6618d4a2eb470e7b2756964f6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/4ce6ea0457d84537b1deddf94dc7129c] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=48.9 K 2024-11-07T12:38:23,584 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fc20854eddbd413c8be86294366de49e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5f882e859b784a91899b291fdbe6a39d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/851b81ed35f54ffb8fed8d5a851870c6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/2488ec52cc5e4dba81a012f927bf9f8c] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=48.9 K 2024-11-07T12:38:23,584 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e428e9be57b4290a81996c2d6ccdc5d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983097918 2024-11-07T12:38:23,584 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc20854eddbd413c8be86294366de49e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983097918 2024-11-07T12:38:23,584 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c44ba4cd23940e2a53d7b98451ed598, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983098671 2024-11-07T12:38:23,584 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f882e859b784a91899b291fdbe6a39d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983098671 2024-11-07T12:38:23,585 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 495538a6618d4a2eb470e7b2756964f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730983099938 2024-11-07T12:38:23,585 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 851b81ed35f54ffb8fed8d5a851870c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730983099938 2024-11-07T12:38:23,585 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce6ea0457d84537b1deddf94dc7129c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983101175 2024-11-07T12:38:23,585 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2488ec52cc5e4dba81a012f927bf9f8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983101175 2024-11-07T12:38:23,592 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#A#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:23,593 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#B#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:23,593 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/19b66795e8f94238a9e43ed035d1fdb4 is 50, key is test_row_0/A:col10/1730983101175/Put/seqid=0 2024-11-07T12:38:23,593 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/6135327dd964417c8fb821c681c48255 is 50, key is test_row_0/B:col10/1730983101175/Put/seqid=0 2024-11-07T12:38:23,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742406_1582 (size=13289) 2024-11-07T12:38:23,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742407_1583 (size=13289) 2024-11-07T12:38:24,004 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/19b66795e8f94238a9e43ed035d1fdb4 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/19b66795e8f94238a9e43ed035d1fdb4 2024-11-07T12:38:24,004 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/6135327dd964417c8fb821c681c48255 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/6135327dd964417c8fb821c681c48255 2024-11-07T12:38:24,008 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/A of 47da5791582e51cd4fb9bc05a2daaec4 into 19b66795e8f94238a9e43ed035d1fdb4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:24,008 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/B of 47da5791582e51cd4fb9bc05a2daaec4 into 6135327dd964417c8fb821c681c48255(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:24,008 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:24,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:24,008 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/B, priority=12, startTime=1730983103583; duration=0sec 2024-11-07T12:38:24,008 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/A, priority=12, startTime=1730983103582; duration=0sec 2024-11-07T12:38:24,008 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:24,008 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:A 2024-11-07T12:38:24,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:24,008 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:B 2024-11-07T12:38:24,008 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:24,009 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:24,009 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): 47da5791582e51cd4fb9bc05a2daaec4/C is initiating minor compaction (all files) 2024-11-07T12:38:24,009 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47da5791582e51cd4fb9bc05a2daaec4/C in TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:24,009 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/fc630d4f98ba470da0662310bede18a6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f10220d88d0c4187b1a5e9917732e97f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75b65bec563a4169b935eaa31ad949fa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/aca443223eb047b9ac63a789bd2e1b36] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp, totalSize=48.9 K 2024-11-07T12:38:24,009 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc630d4f98ba470da0662310bede18a6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983097918 2024-11-07T12:38:24,009 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10220d88d0c4187b1a5e9917732e97f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983098671 2024-11-07T12:38:24,010 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b65bec563a4169b935eaa31ad949fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730983099938 2024-11-07T12:38:24,010 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting aca443223eb047b9ac63a789bd2e1b36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983101175 2024-11-07T12:38:24,017 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47da5791582e51cd4fb9bc05a2daaec4#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:24,018 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/5ccc522575ec404abbbe81aaddd77769 is 50, key is test_row_0/C:col10/1730983101175/Put/seqid=0 2024-11-07T12:38:24,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742408_1584 (size=13289) 2024-11-07T12:38:24,426 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/5ccc522575ec404abbbe81aaddd77769 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5ccc522575ec404abbbe81aaddd77769 2024-11-07T12:38:24,430 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47da5791582e51cd4fb9bc05a2daaec4/C of 47da5791582e51cd4fb9bc05a2daaec4 into 5ccc522575ec404abbbe81aaddd77769(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:24,430 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47da5791582e51cd4fb9bc05a2daaec4: 2024-11-07T12:38:24,430 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., storeName=47da5791582e51cd4fb9bc05a2daaec4/C, priority=12, startTime=1730983103583; duration=0sec 2024-11-07T12:38:24,430 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:24,430 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47da5791582e51cd4fb9bc05a2daaec4:C 2024-11-07T12:38:27,190 INFO [Thread-2194 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2194. 2024-11-07T12:38:27,190 DEBUG [Thread-2194 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:27,190 DEBUG [Thread-2194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:27,190 INFO [Thread-2196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2196. 2024-11-07T12:38:27,190 DEBUG [Thread-2196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:27,190 DEBUG [Thread-2196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:27,190 DEBUG [Thread-2194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:27,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:27,190 DEBUG [Thread-2196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1053 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3158 rows 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1050 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3150 rows 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1048 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3140 rows 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1054 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3162 rows 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1044 2024-11-07T12:38:27,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3132 rows 2024-11-07T12:38:27,191 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:38:27,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:27,191 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:38:27,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:27,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:27,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:27,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T12:38:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:27,196 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983107195"}]},"ts":"1730983107195"} 2024-11-07T12:38:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T12:38:27,199 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:38:27,200 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:38:27,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:38:27,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, UNASSIGN}] 2024-11-07T12:38:27,203 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, UNASSIGN 2024-11-07T12:38:27,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=47da5791582e51cd4fb9bc05a2daaec4, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:27,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, UNASSIGN because future has completed 2024-11-07T12:38:27,205 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:38:27,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:38:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T12:38:27,364 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:27,364 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:38:27,365 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 47da5791582e51cd4fb9bc05a2daaec4, disabling compactions & flushes 2024-11-07T12:38:27,365 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:27,365 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:27,365 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. after waiting 0 ms 2024-11-07T12:38:27,365 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:27,365 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing 47da5791582e51cd4fb9bc05a2daaec4 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T12:38:27,374 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b48fb0d7e461420cab603ba9bee15dba is 50, key is test_row_1/A:col10/1730983107188/Put/seqid=0 2024-11-07T12:38:27,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742409_1585 (size=9857) 2024-11-07T12:38:27,393 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b48fb0d7e461420cab603ba9bee15dba 2024-11-07T12:38:27,403 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/c2d2aef6d97c45a2a382fcde10c05dab is 50, key is test_row_1/B:col10/1730983107188/Put/seqid=0 2024-11-07T12:38:27,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742410_1586 (size=9857) 2024-11-07T12:38:27,422 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/c2d2aef6d97c45a2a382fcde10c05dab 2024-11-07T12:38:27,482 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7edece581bf64f58bc7bceb4e7553779 is 50, key is test_row_1/C:col10/1730983107188/Put/seqid=0 2024-11-07T12:38:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T12:38:27,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742411_1587 (size=9857) 2024-11-07T12:38:27,536 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7edece581bf64f58bc7bceb4e7553779 2024-11-07T12:38:27,548 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/A/b48fb0d7e461420cab603ba9bee15dba as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b48fb0d7e461420cab603ba9bee15dba 2024-11-07T12:38:27,570 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b48fb0d7e461420cab603ba9bee15dba, entries=100, sequenceid=437, filesize=9.6 K 2024-11-07T12:38:27,573 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/B/c2d2aef6d97c45a2a382fcde10c05dab as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c2d2aef6d97c45a2a382fcde10c05dab 2024-11-07T12:38:27,581 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c2d2aef6d97c45a2a382fcde10c05dab, entries=100, sequenceid=437, filesize=9.6 K 2024-11-07T12:38:27,582 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/.tmp/C/7edece581bf64f58bc7bceb4e7553779 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7edece581bf64f58bc7bceb4e7553779 2024-11-07T12:38:27,585 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7edece581bf64f58bc7bceb4e7553779, entries=100, sequenceid=437, filesize=9.6 K 2024-11-07T12:38:27,586 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 47da5791582e51cd4fb9bc05a2daaec4 in 221ms, sequenceid=437, compaction requested=false 2024-11-07T12:38:27,588 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/a1e4636af4224698b9b9d6ed9c3401fe, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4a6342f3f18e470297d2247b8141b15d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/bb017907ef2d49cf88cab0bfd660e3c4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/47500dbeb6bd4e62bfdb7c5bb50bce16, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fd31e6470b9e4cdb828b5e30a23d9edd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6876bdad98cf45f8b80312747fa619b6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/83ed3835b24544b59916722361820457, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6aa52d78a9934c2283610cc43b9c0ba8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b03e08b393014ac7b749c836edd22cf0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/89d68ca1cab5422a967e653da886f6bc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c12880cdc8b347b498af030021c64b06, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/1714d81242a54aa2814d162f83d52174, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b1141b9ea2e94235be0f8066d3aa4eaf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/0886f485c4554fa1b7edfc2e8c827922, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/3c656f101f754512aed00aad412c7d40, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/591e04385b4647a799095114e9dfd1b4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/521adac2eafc4de199aaa3a1607d6051, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5a82398a218b474ebb745a31ea787868, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b49bc13134ae45a58d3039ef0d368d62, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b5a1c1ee5c0749ff96485e87a334fd1d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/7388b63baff74483815234190008c030, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c2e500cd0882405bb397dbdc2c5df957, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/41b74780d8d747c1a8dde23b9bfb0647, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4d0bebd8d5054af69202125eb1309756, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/002eb2ed2f364ad08bf074d7e3d1aae8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fc20854eddbd413c8be86294366de49e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5f882e859b784a91899b291fdbe6a39d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/851b81ed35f54ffb8fed8d5a851870c6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/2488ec52cc5e4dba81a012f927bf9f8c] to archive 2024-11-07T12:38:27,589 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:38:27,594 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/a1e4636af4224698b9b9d6ed9c3401fe to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/a1e4636af4224698b9b9d6ed9c3401fe 2024-11-07T12:38:27,595 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4a6342f3f18e470297d2247b8141b15d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4a6342f3f18e470297d2247b8141b15d 2024-11-07T12:38:27,596 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/bb017907ef2d49cf88cab0bfd660e3c4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/bb017907ef2d49cf88cab0bfd660e3c4 2024-11-07T12:38:27,597 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/47500dbeb6bd4e62bfdb7c5bb50bce16 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/47500dbeb6bd4e62bfdb7c5bb50bce16 2024-11-07T12:38:27,598 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fd31e6470b9e4cdb828b5e30a23d9edd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fd31e6470b9e4cdb828b5e30a23d9edd 2024-11-07T12:38:27,599 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6876bdad98cf45f8b80312747fa619b6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6876bdad98cf45f8b80312747fa619b6 2024-11-07T12:38:27,602 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/83ed3835b24544b59916722361820457 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/83ed3835b24544b59916722361820457 2024-11-07T12:38:27,602 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6aa52d78a9934c2283610cc43b9c0ba8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/6aa52d78a9934c2283610cc43b9c0ba8 2024-11-07T12:38:27,603 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b03e08b393014ac7b749c836edd22cf0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b03e08b393014ac7b749c836edd22cf0 2024-11-07T12:38:27,604 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/89d68ca1cab5422a967e653da886f6bc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/89d68ca1cab5422a967e653da886f6bc 2024-11-07T12:38:27,605 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c12880cdc8b347b498af030021c64b06 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c12880cdc8b347b498af030021c64b06 2024-11-07T12:38:27,606 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/1714d81242a54aa2814d162f83d52174 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/1714d81242a54aa2814d162f83d52174 2024-11-07T12:38:27,607 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b1141b9ea2e94235be0f8066d3aa4eaf to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b1141b9ea2e94235be0f8066d3aa4eaf 2024-11-07T12:38:27,608 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/0886f485c4554fa1b7edfc2e8c827922 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/0886f485c4554fa1b7edfc2e8c827922 2024-11-07T12:38:27,609 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/3c656f101f754512aed00aad412c7d40 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/3c656f101f754512aed00aad412c7d40 2024-11-07T12:38:27,610 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/591e04385b4647a799095114e9dfd1b4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/591e04385b4647a799095114e9dfd1b4 2024-11-07T12:38:27,611 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/521adac2eafc4de199aaa3a1607d6051 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/521adac2eafc4de199aaa3a1607d6051 2024-11-07T12:38:27,612 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5a82398a218b474ebb745a31ea787868 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5a82398a218b474ebb745a31ea787868 2024-11-07T12:38:27,614 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b49bc13134ae45a58d3039ef0d368d62 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b49bc13134ae45a58d3039ef0d368d62 2024-11-07T12:38:27,615 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b5a1c1ee5c0749ff96485e87a334fd1d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b5a1c1ee5c0749ff96485e87a334fd1d 2024-11-07T12:38:27,616 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/7388b63baff74483815234190008c030 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/7388b63baff74483815234190008c030 2024-11-07T12:38:27,617 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c2e500cd0882405bb397dbdc2c5df957 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/c2e500cd0882405bb397dbdc2c5df957 2024-11-07T12:38:27,623 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/41b74780d8d747c1a8dde23b9bfb0647 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/41b74780d8d747c1a8dde23b9bfb0647 2024-11-07T12:38:27,624 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4d0bebd8d5054af69202125eb1309756 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/4d0bebd8d5054af69202125eb1309756 2024-11-07T12:38:27,626 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/002eb2ed2f364ad08bf074d7e3d1aae8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/002eb2ed2f364ad08bf074d7e3d1aae8 2024-11-07T12:38:27,628 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fc20854eddbd413c8be86294366de49e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/fc20854eddbd413c8be86294366de49e 2024-11-07T12:38:27,629 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5f882e859b784a91899b291fdbe6a39d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/5f882e859b784a91899b291fdbe6a39d 2024-11-07T12:38:27,631 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/851b81ed35f54ffb8fed8d5a851870c6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/851b81ed35f54ffb8fed8d5a851870c6 2024-11-07T12:38:27,632 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/2488ec52cc5e4dba81a012f927bf9f8c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/2488ec52cc5e4dba81a012f927bf9f8c 2024-11-07T12:38:27,637 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/1924a0c5d1004edcab1bbf73507a4b3e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/bf437d2d93684fa9b29f08408a113622, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c93f67bc111a4520a953f82da3b6de48, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c505b1425aef4812a595da727b23b2a9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/62e880295965454ca99637b003e59ca4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/78466839c4214d769e93a4aca3920723, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9bff83c8e17d4757bc5964f446ba68d7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/eddf70ff6bea45b388bfb42c8ca5ccb8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/051d31aac1b240cbbaba66f867f59c95, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8179e9a9fda643ff9faf31af68581674, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/0a6bff407ddd4c36b4d8246b9737f848, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/7f28ea9bd6684d2caca4f54c8db261cf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e4a91de6cc64fb1b96394fc2c3b17a2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/5bf50fd49d2043dbbe63841530eff037, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/302bcdcb3374480da8a0a704809d4451, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/75af55c80c004b9f8aeb16e9813740c2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/721e44cd452f4640b3c9e2dec3aae427, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/138645212c514b34848d31752b4253f4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/e2f11acd5fc749278db47aef6ae14398, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/61809c92f7c54ad89d1752e9fb7ca75d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c42c287079f4cc88754673ff9555df8, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/90a813cfbc94416982428655f3980608, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/791e6809e4824df1a60cbd8105c02bac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/628a3de3120d4a7ba4c6899f0456438c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e428e9be57b4290a81996c2d6ccdc5d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/a299a79d962549d7aa33ccb1f1a8ea93, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c44ba4cd23940e2a53d7b98451ed598, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/495538a6618d4a2eb470e7b2756964f6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/4ce6ea0457d84537b1deddf94dc7129c] to archive 2024-11-07T12:38:27,638 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:38:27,640 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/1924a0c5d1004edcab1bbf73507a4b3e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/1924a0c5d1004edcab1bbf73507a4b3e 2024-11-07T12:38:27,641 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/bf437d2d93684fa9b29f08408a113622 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/bf437d2d93684fa9b29f08408a113622 2024-11-07T12:38:27,642 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c93f67bc111a4520a953f82da3b6de48 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c93f67bc111a4520a953f82da3b6de48 2024-11-07T12:38:27,643 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c505b1425aef4812a595da727b23b2a9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c505b1425aef4812a595da727b23b2a9 2024-11-07T12:38:27,644 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/62e880295965454ca99637b003e59ca4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/62e880295965454ca99637b003e59ca4 2024-11-07T12:38:27,648 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/78466839c4214d769e93a4aca3920723 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/78466839c4214d769e93a4aca3920723 2024-11-07T12:38:27,650 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9bff83c8e17d4757bc5964f446ba68d7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9bff83c8e17d4757bc5964f446ba68d7 2024-11-07T12:38:27,651 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/eddf70ff6bea45b388bfb42c8ca5ccb8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/eddf70ff6bea45b388bfb42c8ca5ccb8 2024-11-07T12:38:27,652 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/051d31aac1b240cbbaba66f867f59c95 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/051d31aac1b240cbbaba66f867f59c95 2024-11-07T12:38:27,653 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8179e9a9fda643ff9faf31af68581674 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8179e9a9fda643ff9faf31af68581674 2024-11-07T12:38:27,655 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/0a6bff407ddd4c36b4d8246b9737f848 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/0a6bff407ddd4c36b4d8246b9737f848 2024-11-07T12:38:27,656 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/7f28ea9bd6684d2caca4f54c8db261cf to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/7f28ea9bd6684d2caca4f54c8db261cf 2024-11-07T12:38:27,664 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e4a91de6cc64fb1b96394fc2c3b17a2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e4a91de6cc64fb1b96394fc2c3b17a2 2024-11-07T12:38:27,667 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/5bf50fd49d2043dbbe63841530eff037 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/5bf50fd49d2043dbbe63841530eff037 2024-11-07T12:38:27,668 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/302bcdcb3374480da8a0a704809d4451 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/302bcdcb3374480da8a0a704809d4451 2024-11-07T12:38:27,669 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/75af55c80c004b9f8aeb16e9813740c2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/75af55c80c004b9f8aeb16e9813740c2 2024-11-07T12:38:27,670 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/721e44cd452f4640b3c9e2dec3aae427 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/721e44cd452f4640b3c9e2dec3aae427 2024-11-07T12:38:27,673 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/138645212c514b34848d31752b4253f4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/138645212c514b34848d31752b4253f4 2024-11-07T12:38:27,674 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/e2f11acd5fc749278db47aef6ae14398 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/e2f11acd5fc749278db47aef6ae14398 2024-11-07T12:38:27,675 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/61809c92f7c54ad89d1752e9fb7ca75d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/61809c92f7c54ad89d1752e9fb7ca75d 2024-11-07T12:38:27,677 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c42c287079f4cc88754673ff9555df8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c42c287079f4cc88754673ff9555df8 2024-11-07T12:38:27,678 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/90a813cfbc94416982428655f3980608 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/90a813cfbc94416982428655f3980608 2024-11-07T12:38:27,680 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/791e6809e4824df1a60cbd8105c02bac to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/791e6809e4824df1a60cbd8105c02bac 2024-11-07T12:38:27,687 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/628a3de3120d4a7ba4c6899f0456438c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/628a3de3120d4a7ba4c6899f0456438c 2024-11-07T12:38:27,688 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e428e9be57b4290a81996c2d6ccdc5d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/8e428e9be57b4290a81996c2d6ccdc5d 2024-11-07T12:38:27,690 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/a299a79d962549d7aa33ccb1f1a8ea93 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/a299a79d962549d7aa33ccb1f1a8ea93 2024-11-07T12:38:27,691 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c44ba4cd23940e2a53d7b98451ed598 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/9c44ba4cd23940e2a53d7b98451ed598 2024-11-07T12:38:27,692 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/495538a6618d4a2eb470e7b2756964f6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/495538a6618d4a2eb470e7b2756964f6 2024-11-07T12:38:27,693 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/4ce6ea0457d84537b1deddf94dc7129c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/4ce6ea0457d84537b1deddf94dc7129c 2024-11-07T12:38:27,701 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a2726974b0594542b852b4a1631b17e3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/853f297596f44630bd2235f1e93c773f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/ac96b9be330849bdb9b8cd87ff5c8b9c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/8c0059a9b86348fbab1f3d0ccf6c618b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/bdfba015330f46ca947c2a8b6e94c7ff, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a7b3344fedbf423097d00088d0e9a59a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7323aa031a9e4606af2c282cb0b7e7ba, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7de96ea856b74b639efff1d9c220a09b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/267749ca711a412dbfa11af34f35c613, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75f7bbe9f1874c37a24b032aa7cc9d9d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f6e0797d54f24ac5a91330d2f5fe444a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c657ad4554cf4f48a696318e6784b692, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/1c73e58bafbd42e18813ba4ff27126fe, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/30571bd459944d0db3d0bcc86863fcb4, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/32df2d6455754b409ce4b8d010f03d4e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/73b6e8c7831144388cfdf90784f3d640, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/4b7d53e6cb5545b48ae298642ab1521e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/07c234dae52b4d5192bfee67d6ef11bd, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/eac0b8cdbcdd43c29310ab26a9d47369, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5b43bed3eb224af1ba02945cf936e249, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/9452aa7fb24b458e8c8d3be035b62dd6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/b506603f59614ee6b645982f6a0801cf, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/76a9ab0eb8c24727bdf8edebd313488a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a197f052744e46b3b920bf9a1d7a4ed0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/fc630d4f98ba470da0662310bede18a6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c0690a73566645ec8800cf241f30353f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f10220d88d0c4187b1a5e9917732e97f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75b65bec563a4169b935eaa31ad949fa, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/aca443223eb047b9ac63a789bd2e1b36] to archive 2024-11-07T12:38:27,702 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:38:27,706 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a2726974b0594542b852b4a1631b17e3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a2726974b0594542b852b4a1631b17e3 2024-11-07T12:38:27,707 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/853f297596f44630bd2235f1e93c773f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/853f297596f44630bd2235f1e93c773f 2024-11-07T12:38:27,708 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/ac96b9be330849bdb9b8cd87ff5c8b9c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/ac96b9be330849bdb9b8cd87ff5c8b9c 2024-11-07T12:38:27,710 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/8c0059a9b86348fbab1f3d0ccf6c618b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/8c0059a9b86348fbab1f3d0ccf6c618b 2024-11-07T12:38:27,720 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/bdfba015330f46ca947c2a8b6e94c7ff to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/bdfba015330f46ca947c2a8b6e94c7ff 2024-11-07T12:38:27,730 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a7b3344fedbf423097d00088d0e9a59a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a7b3344fedbf423097d00088d0e9a59a 2024-11-07T12:38:27,744 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7323aa031a9e4606af2c282cb0b7e7ba to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7323aa031a9e4606af2c282cb0b7e7ba 2024-11-07T12:38:27,752 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7de96ea856b74b639efff1d9c220a09b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7de96ea856b74b639efff1d9c220a09b 2024-11-07T12:38:27,759 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/267749ca711a412dbfa11af34f35c613 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/267749ca711a412dbfa11af34f35c613 2024-11-07T12:38:27,761 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75f7bbe9f1874c37a24b032aa7cc9d9d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75f7bbe9f1874c37a24b032aa7cc9d9d 2024-11-07T12:38:27,763 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f6e0797d54f24ac5a91330d2f5fe444a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f6e0797d54f24ac5a91330d2f5fe444a 2024-11-07T12:38:27,764 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c657ad4554cf4f48a696318e6784b692 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c657ad4554cf4f48a696318e6784b692 2024-11-07T12:38:27,769 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/1c73e58bafbd42e18813ba4ff27126fe to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/1c73e58bafbd42e18813ba4ff27126fe 2024-11-07T12:38:27,774 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/30571bd459944d0db3d0bcc86863fcb4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/30571bd459944d0db3d0bcc86863fcb4 2024-11-07T12:38:27,775 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/32df2d6455754b409ce4b8d010f03d4e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/32df2d6455754b409ce4b8d010f03d4e 2024-11-07T12:38:27,776 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/73b6e8c7831144388cfdf90784f3d640 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/73b6e8c7831144388cfdf90784f3d640 2024-11-07T12:38:27,777 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/4b7d53e6cb5545b48ae298642ab1521e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/4b7d53e6cb5545b48ae298642ab1521e 2024-11-07T12:38:27,779 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/07c234dae52b4d5192bfee67d6ef11bd to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/07c234dae52b4d5192bfee67d6ef11bd 2024-11-07T12:38:27,780 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/eac0b8cdbcdd43c29310ab26a9d47369 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/eac0b8cdbcdd43c29310ab26a9d47369 2024-11-07T12:38:27,796 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5b43bed3eb224af1ba02945cf936e249 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5b43bed3eb224af1ba02945cf936e249 2024-11-07T12:38:27,800 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/9452aa7fb24b458e8c8d3be035b62dd6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/9452aa7fb24b458e8c8d3be035b62dd6 2024-11-07T12:38:27,801 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/b506603f59614ee6b645982f6a0801cf to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/b506603f59614ee6b645982f6a0801cf 2024-11-07T12:38:27,802 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/76a9ab0eb8c24727bdf8edebd313488a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/76a9ab0eb8c24727bdf8edebd313488a 2024-11-07T12:38:27,803 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a197f052744e46b3b920bf9a1d7a4ed0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/a197f052744e46b3b920bf9a1d7a4ed0 2024-11-07T12:38:27,804 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/fc630d4f98ba470da0662310bede18a6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/fc630d4f98ba470da0662310bede18a6 2024-11-07T12:38:27,805 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c0690a73566645ec8800cf241f30353f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/c0690a73566645ec8800cf241f30353f 2024-11-07T12:38:27,807 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f10220d88d0c4187b1a5e9917732e97f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/f10220d88d0c4187b1a5e9917732e97f 2024-11-07T12:38:27,808 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75b65bec563a4169b935eaa31ad949fa to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/75b65bec563a4169b935eaa31ad949fa 2024-11-07T12:38:27,813 DEBUG [StoreCloser-TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/aca443223eb047b9ac63a789bd2e1b36 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/aca443223eb047b9ac63a789bd2e1b36 2024-11-07T12:38:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T12:38:27,859 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/recovered.edits/440.seqid, newMaxSeqId=440, maxSeqId=1 2024-11-07T12:38:27,866 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. 2024-11-07T12:38:27,866 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 47da5791582e51cd4fb9bc05a2daaec4: Waiting for close lock at 1730983107365Running coprocessor pre-close hooks at 1730983107365Disabling compacts and flushes for region at 1730983107365Disabling writes for close at 1730983107365Obtaining lock to block concurrent updates at 1730983107365Preparing flush snapshotting stores in 47da5791582e51cd4fb9bc05a2daaec4 at 1730983107365Finished memstore snapshotting TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730983107365Flushing stores of TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4. at 1730983107366 (+1 ms)Flushing 47da5791582e51cd4fb9bc05a2daaec4/A: creating writer at 1730983107366Flushing 47da5791582e51cd4fb9bc05a2daaec4/A: appending metadata at 1730983107373 (+7 ms)Flushing 47da5791582e51cd4fb9bc05a2daaec4/A: closing flushed file at 1730983107373Flushing 47da5791582e51cd4fb9bc05a2daaec4/B: creating writer at 1730983107400 (+27 ms)Flushing 47da5791582e51cd4fb9bc05a2daaec4/B: appending metadata at 1730983107403 (+3 ms)Flushing 47da5791582e51cd4fb9bc05a2daaec4/B: closing flushed file at 1730983107403Flushing 47da5791582e51cd4fb9bc05a2daaec4/C: creating writer at 1730983107457 (+54 ms)Flushing 47da5791582e51cd4fb9bc05a2daaec4/C: appending metadata at 1730983107482 (+25 ms)Flushing 47da5791582e51cd4fb9bc05a2daaec4/C: closing flushed file at 1730983107482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70d958b1: reopening flushed file at 1730983107545 (+63 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7270a2e: reopening flushed file at 1730983107570 (+25 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bbbf863: reopening flushed file at 1730983107581 (+11 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 47da5791582e51cd4fb9bc05a2daaec4 in 221ms, sequenceid=437, compaction requested=false at 1730983107586 (+5 ms)Writing region close event to WAL at 1730983107821 (+235 ms)Running coprocessor post-close hooks at 1730983107866 (+45 ms)Closed at 1730983107866 2024-11-07T12:38:27,869 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:27,870 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=47da5791582e51cd4fb9bc05a2daaec4, regionState=CLOSED 2024-11-07T12:38:27,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:38:27,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-07T12:38:27,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 47da5791582e51cd4fb9bc05a2daaec4, server=87fb68174c52,38515,1730982961782 in 671 msec 2024-11-07T12:38:27,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-07T12:38:27,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47da5791582e51cd4fb9bc05a2daaec4, UNASSIGN in 677 msec 2024-11-07T12:38:27,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-07T12:38:27,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 681 msec 2024-11-07T12:38:27,885 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983107885"}]},"ts":"1730983107885"} 2024-11-07T12:38:27,890 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:38:27,890 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:38:27,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 699 msec 2024-11-07T12:38:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T12:38:28,338 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:28,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T12:38:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:28,341 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-07T12:38:28,343 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:28,346 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:28,348 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/recovered.edits] 2024-11-07T12:38:28,351 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/19b66795e8f94238a9e43ed035d1fdb4 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/19b66795e8f94238a9e43ed035d1fdb4 2024-11-07T12:38:28,353 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b48fb0d7e461420cab603ba9bee15dba to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/A/b48fb0d7e461420cab603ba9bee15dba 2024-11-07T12:38:28,355 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/6135327dd964417c8fb821c681c48255 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/6135327dd964417c8fb821c681c48255 2024-11-07T12:38:28,356 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c2d2aef6d97c45a2a382fcde10c05dab to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/B/c2d2aef6d97c45a2a382fcde10c05dab 2024-11-07T12:38:28,358 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5ccc522575ec404abbbe81aaddd77769 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/5ccc522575ec404abbbe81aaddd77769 2024-11-07T12:38:28,359 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7edece581bf64f58bc7bceb4e7553779 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/C/7edece581bf64f58bc7bceb4e7553779 2024-11-07T12:38:28,361 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/recovered.edits/440.seqid to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4/recovered.edits/440.seqid 2024-11-07T12:38:28,362 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/47da5791582e51cd4fb9bc05a2daaec4 2024-11-07T12:38:28,362 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:38:28,364 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:28,370 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:38:28,373 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:38:28,374 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:28,375 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:38:28,375 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983108375"}]},"ts":"9223372036854775807"} 2024-11-07T12:38:28,378 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:38:28,378 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 47da5791582e51cd4fb9bc05a2daaec4, NAME => 'TestAcidGuarantees,,1730983079467.47da5791582e51cd4fb9bc05a2daaec4.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:38:28,378 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:38:28,378 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983108378"}]},"ts":"9223372036854775807"} 2024-11-07T12:38:28,381 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:38:28,385 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:28,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-07T12:38:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-07T12:38:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:38:28,449 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:28,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 243), OpenFileDescriptor=451 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 447) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6498 (was 6832) 2024-11-07T12:38:28,468 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=6498 2024-11-07T12:38:28,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:38:28,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:38:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:28,471 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:38:28,471 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:28,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-07T12:38:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:38:28,472 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:38:28,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742412_1588 (size=959) 2024-11-07T12:38:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:38:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:38:28,879 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e 2024-11-07T12:38:28,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742413_1589 (size=53) 2024-11-07T12:38:28,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:38:28,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bc1940ad44a4ffd987a2f6164472fdb6, disabling compactions & flushes 2024-11-07T12:38:28,885 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:28,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:28,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. after waiting 0 ms 2024-11-07T12:38:28,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:28,885 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:28,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bc1940ad44a4ffd987a2f6164472fdb6: Waiting for close lock at 1730983108885Disabling compacts and flushes for region at 1730983108885Disabling writes for close at 1730983108885Writing region close event to WAL at 1730983108885Closed at 1730983108885 2024-11-07T12:38:28,888 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:38:28,888 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983108888"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983108888"}]},"ts":"1730983108888"} 2024-11-07T12:38:28,893 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:38:28,894 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:38:28,894 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983108894"}]},"ts":"1730983108894"} 2024-11-07T12:38:28,895 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:38:28,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, ASSIGN}] 2024-11-07T12:38:28,897 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, ASSIGN 2024-11-07T12:38:28,897 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, ASSIGN; state=OFFLINE, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=false 2024-11-07T12:38:29,048 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=bc1940ad44a4ffd987a2f6164472fdb6, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:29,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, ASSIGN because future has completed 2024-11-07T12:38:29,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:38:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:38:29,206 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:29,206 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:38:29,206 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,207 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:38:29,207 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,207 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,213 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,215 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc1940ad44a4ffd987a2f6164472fdb6 columnFamilyName A 2024-11-07T12:38:29,215 DEBUG [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:29,215 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(327): Store=bc1940ad44a4ffd987a2f6164472fdb6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:29,215 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,225 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc1940ad44a4ffd987a2f6164472fdb6 columnFamilyName B 2024-11-07T12:38:29,225 DEBUG [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:29,227 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(327): Store=bc1940ad44a4ffd987a2f6164472fdb6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:29,227 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,232 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc1940ad44a4ffd987a2f6164472fdb6 columnFamilyName C 2024-11-07T12:38:29,232 DEBUG [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:29,233 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(327): Store=bc1940ad44a4ffd987a2f6164472fdb6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:29,233 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,233 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,234 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,236 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,236 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,236 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:38:29,238 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,241 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:38:29,241 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened bc1940ad44a4ffd987a2f6164472fdb6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74863318, jitterRate=0.11555036902427673}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:38:29,241 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:29,242 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for bc1940ad44a4ffd987a2f6164472fdb6: Running coprocessor pre-open hook at 1730983109207Writing region info on filesystem at 1730983109207Initializing all the Stores at 1730983109213 (+6 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983109213Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983109213Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983109213Cleaning up temporary data from old regions at 1730983109236 (+23 ms)Running coprocessor post-open hooks at 1730983109241 (+5 ms)Region opened successfully at 1730983109242 (+1 ms) 2024-11-07T12:38:29,243 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., pid=154, masterSystemTime=1730983109202 2024-11-07T12:38:29,249 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:29,249 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:29,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=bc1940ad44a4ffd987a2f6164472fdb6, regionState=OPEN, openSeqNum=2, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:29,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:38:29,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-07T12:38:29,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 in 211 msec 2024-11-07T12:38:29,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-07T12:38:29,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, ASSIGN in 369 msec 2024-11-07T12:38:29,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:38:29,269 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983109268"}]},"ts":"1730983109268"} 2024-11-07T12:38:29,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:38:29,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:38:29,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 802 msec 2024-11-07T12:38:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:38:29,608 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:29,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:29,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:29,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:29,613 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4285c54f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:29,619 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d2101f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:29,620 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:29,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:29,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:38:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 87fb68174c52,36251,1730982961115 2024-11-07T12:38:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e2ed103 2024-11-07T12:38:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:38:29,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:38:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:38:29,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:38:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:29,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742414_1590 (size=995) 2024-11-07T12:38:30,099 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T12:38:30,099 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T12:38:30,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:38:30,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, REOPEN/MOVE}] 2024-11-07T12:38:30,108 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, REOPEN/MOVE 2024-11-07T12:38:30,109 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=bc1940ad44a4ffd987a2f6164472fdb6, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, REOPEN/MOVE because future has completed 2024-11-07T12:38:30,111 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:38:30,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:38:30,247 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:38:30,264 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,264 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:38:30,264 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing bc1940ad44a4ffd987a2f6164472fdb6, disabling compactions & flushes 2024-11-07T12:38:30,264 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,264 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,264 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. after waiting 0 ms 2024-11-07T12:38:30,265 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,268 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:38:30,269 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,269 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for bc1940ad44a4ffd987a2f6164472fdb6: Waiting for close lock at 1730983110264Running coprocessor pre-close hooks at 1730983110264Disabling compacts and flushes for region at 1730983110264Disabling writes for close at 1730983110264Writing region close event to WAL at 1730983110265 (+1 ms)Running coprocessor post-close hooks at 1730983110269 (+4 ms)Closed at 1730983110269 2024-11-07T12:38:30,269 WARN [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: bc1940ad44a4ffd987a2f6164472fdb6 to self. 2024-11-07T12:38:30,271 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,271 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=bc1940ad44a4ffd987a2f6164472fdb6, regionState=CLOSED 2024-11-07T12:38:30,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:38:30,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-07T12:38:30,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 in 162 msec 2024-11-07T12:38:30,275 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, REOPEN/MOVE; state=CLOSED, location=87fb68174c52,38515,1730982961782; forceNewPlan=false, retain=true 2024-11-07T12:38:30,425 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=bc1940ad44a4ffd987a2f6164472fdb6, regionState=OPENING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, REOPEN/MOVE because future has completed 2024-11-07T12:38:30,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:38:30,581 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,581 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:38:30,581 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,581 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:38:30,581 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,581 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,582 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,583 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc1940ad44a4ffd987a2f6164472fdb6 columnFamilyName A 2024-11-07T12:38:30,584 DEBUG [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:30,584 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(327): Store=bc1940ad44a4ffd987a2f6164472fdb6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:30,585 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,585 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc1940ad44a4ffd987a2f6164472fdb6 columnFamilyName B 2024-11-07T12:38:30,585 DEBUG [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:30,585 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(327): Store=bc1940ad44a4ffd987a2f6164472fdb6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:30,585 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,586 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc1940ad44a4ffd987a2f6164472fdb6 columnFamilyName C 2024-11-07T12:38:30,586 DEBUG [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:30,586 INFO [StoreOpener-bc1940ad44a4ffd987a2f6164472fdb6-1 {}] regionserver.HStore(327): Store=bc1940ad44a4ffd987a2f6164472fdb6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:38:30,586 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,587 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,587 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,588 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,588 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,588 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:38:30,589 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,590 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened bc1940ad44a4ffd987a2f6164472fdb6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74915759, jitterRate=0.1163318008184433}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:38:30,590 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,590 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for bc1940ad44a4ffd987a2f6164472fdb6: Running coprocessor pre-open hook at 1730983110582Writing region info on filesystem at 1730983110582Initializing all the Stores at 1730983110582Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983110582Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983110582Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983110582Cleaning up temporary data from old regions at 1730983110588 (+6 ms)Running coprocessor post-open hooks at 1730983110590 (+2 ms)Region opened successfully at 1730983110590 2024-11-07T12:38:30,591 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., pid=159, masterSystemTime=1730983110579 2024-11-07T12:38:30,592 DEBUG [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,592 INFO [RS_OPEN_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=bc1940ad44a4ffd987a2f6164472fdb6, regionState=OPEN, openSeqNum=5, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:38:30,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-07T12:38:30,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 in 168 msec 2024-11-07T12:38:30,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-07T12:38:30,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, REOPEN/MOVE in 489 msec 2024-11-07T12:38:30,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-07T12:38:30,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 496 msec 2024-11-07T12:38:30,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 961 msec 2024-11-07T12:38:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-07T12:38:30,602 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:30,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,603 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ccda3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,604 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49579017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,606 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582735a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,607 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b2af7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,608 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60887624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,609 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24914e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,611 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b826e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,612 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e05047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,613 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2d947c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,614 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d25665a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,616 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ecafcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,617 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405c73ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,618 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,618 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,618 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34bc880a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,619 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6c11fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,621 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d373b2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,622 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644d7f2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,623 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b76b5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd5225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87fb68174c52,36251,-1 for getting cluster id 2024-11-07T12:38:30,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:38:30,626 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55d9ff63-aba5-4d8c-b02d-9ea864170a03' 2024-11-07T12:38:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:38:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55d9ff63-aba5-4d8c-b02d-9ea864170a03" 2024-11-07T12:38:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79784bbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87fb68174c52,36251,-1] 2024-11-07T12:38:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:38:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:30,627 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:38:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7adf3842, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:38:30,628 DEBUG [Thread-2610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,628 DEBUG [Thread-2602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,628 DEBUG [Thread-2608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,629 DEBUG [Thread-2604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,629 DEBUG [Thread-2621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,631 DEBUG [Thread-2621 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,631 DEBUG [Thread-2606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-07T12:38:30,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:38:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,632 DEBUG [Thread-2606 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,632 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:30,633 DEBUG [Thread-2606 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,633 DEBUG [Thread-2617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,634 DEBUG [Thread-2615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,634 DEBUG [Thread-2613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,635 DEBUG [Thread-2613 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,635 DEBUG [Thread-2613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,635 DEBUG [Thread-2619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:38:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,636 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:30,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:30,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87fb68174c52,38515,1730982961782, seqNum=-1] 2024-11-07T12:38:30,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:38:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:38:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5] 2024-11-07T12:38:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:30,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:38:30,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f11a7cfc528a48f88146a9932403d4cb_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983110637/Put/seqid=0 2024-11-07T12:38:30,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742415_1591 (size=12154) 2024-11-07T12:38:30,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983170682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983170682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983170689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983170689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983170689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:38:30,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:30,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983170798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983170798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983170798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983170798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983170798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:38:30,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:30,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:30,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:30,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:30,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983171007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983171007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983171007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983171008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983171008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:31,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:31,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:31,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,137 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f11a7cfc528a48f88146a9932403d4cb_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f11a7cfc528a48f88146a9932403d4cb_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:31,145 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ad8ecfccd4244863962fde650bae818a, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:31,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ad8ecfccd4244863962fde650bae818a is 175, key is test_row_0/A:col10/1730983110637/Put/seqid=0 2024-11-07T12:38:31,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742416_1592 (size=30955) 2024-11-07T12:38:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:38:31,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:31,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:31,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983171318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983171318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983171318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983171318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983171318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:31,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:31,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:31,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,594 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ad8ecfccd4244863962fde650bae818a 2024-11-07T12:38:31,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/58f831156d8a458dbaa08c172add31ca is 50, key is test_row_0/B:col10/1730983110637/Put/seqid=0 2024-11-07T12:38:31,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742417_1593 (size=12001) 2024-11-07T12:38:31,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/58f831156d8a458dbaa08c172add31ca 2024-11-07T12:38:31,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/37f4313a45284e3580bbbab46d4828d2 is 50, key is test_row_0/C:col10/1730983110637/Put/seqid=0 2024-11-07T12:38:31,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742418_1594 (size=12001) 2024-11-07T12:38:31,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/37f4313a45284e3580bbbab46d4828d2 2024-11-07T12:38:31,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ad8ecfccd4244863962fde650bae818a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad8ecfccd4244863962fde650bae818a 2024-11-07T12:38:31,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad8ecfccd4244863962fde650bae818a, entries=150, sequenceid=15, filesize=30.2 K 2024-11-07T12:38:31,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/58f831156d8a458dbaa08c172add31ca as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58f831156d8a458dbaa08c172add31ca 2024-11-07T12:38:31,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58f831156d8a458dbaa08c172add31ca, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T12:38:31,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/37f4313a45284e3580bbbab46d4828d2 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/37f4313a45284e3580bbbab46d4828d2 2024-11-07T12:38:31,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/37f4313a45284e3580bbbab46d4828d2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T12:38:31,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for bc1940ad44a4ffd987a2f6164472fdb6 in 1115ms, sequenceid=15, compaction requested=false 2024-11-07T12:38:31,763 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T12:38:31,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:38:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:31,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:31,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T12:38:31,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983171833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983171833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983171834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983171838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983171837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fdeaef24ef4a424ea45a12937cded766_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983111829/Put/seqid=0 2024-11-07T12:38:31,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742419_1595 (size=12154) 2024-11-07T12:38:31,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:31,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fdeaef24ef4a424ea45a12937cded766_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdeaef24ef4a424ea45a12937cded766_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:31,854 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/af9b509181da4071a219ef918f82318e, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:31,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/af9b509181da4071a219ef918f82318e is 175, key is test_row_0/A:col10/1730983111829/Put/seqid=0 2024-11-07T12:38:31,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742420_1596 (size=30955) 2024-11-07T12:38:31,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/af9b509181da4071a219ef918f82318e 2024-11-07T12:38:31,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/6828809ce7b84b9090142eabb6a75a45 is 50, key is test_row_0/B:col10/1730983111829/Put/seqid=0 2024-11-07T12:38:31,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:31,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:31,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742421_1597 (size=12001) 2024-11-07T12:38:31,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983171947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983171947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983171948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983171948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983171948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:32,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:32,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983172157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983172157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983172158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983172158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983172160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:32,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:32,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/6828809ce7b84b9090142eabb6a75a45 2024-11-07T12:38:32,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d10d2628dae14bc584a9000bc3ee5293 is 50, key is test_row_0/C:col10/1730983111829/Put/seqid=0 2024-11-07T12:38:32,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742422_1598 (size=12001) 2024-11-07T12:38:32,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d10d2628dae14bc584a9000bc3ee5293 2024-11-07T12:38:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/af9b509181da4071a219ef918f82318e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/af9b509181da4071a219ef918f82318e 2024-11-07T12:38:32,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983172468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983172468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983172468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983172469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983172469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:32,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/af9b509181da4071a219ef918f82318e, entries=150, sequenceid=43, filesize=30.2 K 2024-11-07T12:38:32,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/6828809ce7b84b9090142eabb6a75a45 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/6828809ce7b84b9090142eabb6a75a45 2024-11-07T12:38:32,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:32,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:32,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/6828809ce7b84b9090142eabb6a75a45, entries=150, sequenceid=43, filesize=11.7 K 2024-11-07T12:38:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d10d2628dae14bc584a9000bc3ee5293 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d10d2628dae14bc584a9000bc3ee5293 2024-11-07T12:38:32,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d10d2628dae14bc584a9000bc3ee5293, entries=150, sequenceid=43, filesize=11.7 K 2024-11-07T12:38:32,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for bc1940ad44a4ffd987a2f6164472fdb6 in 711ms, sequenceid=43, compaction requested=false 2024-11-07T12:38:32,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:32,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:38:32,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:32,682 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T12:38:32,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076a2ca1d316714b1eae149b76d18b91d9_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983111835/Put/seqid=0 2024-11-07T12:38:32,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742423_1599 (size=12154) 2024-11-07T12:38:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:32,753 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076a2ca1d316714b1eae149b76d18b91d9_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a2ca1d316714b1eae149b76d18b91d9_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/028767d2eb714363b3956fef5581d3ac, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:32,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/028767d2eb714363b3956fef5581d3ac is 175, key is test_row_0/A:col10/1730983111835/Put/seqid=0 2024-11-07T12:38:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:38:32,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742424_1600 (size=30955) 2024-11-07T12:38:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:32,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:33,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983173016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983173018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983173020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983173021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983173021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983173128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983173128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983173129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983173138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983173138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,202 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/028767d2eb714363b3956fef5581d3ac 2024-11-07T12:38:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/d1badeeead804229bf1ea4357cdeb68d is 50, key is test_row_0/B:col10/1730983111835/Put/seqid=0 2024-11-07T12:38:33,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742425_1601 (size=12001) 2024-11-07T12:38:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983173337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983173338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983173338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983173347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983173348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,411 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:38:33,614 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/d1badeeead804229bf1ea4357cdeb68d 2024-11-07T12:38:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d4c272987ae44ec7b1e8e3a108692a2a is 50, key is test_row_0/C:col10/1730983111835/Put/seqid=0 2024-11-07T12:38:33,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983173647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983173648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983173648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742426_1602 (size=12001) 2024-11-07T12:38:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983173657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983173657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,054 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d4c272987ae44ec7b1e8e3a108692a2a 2024-11-07T12:38:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/028767d2eb714363b3956fef5581d3ac as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/028767d2eb714363b3956fef5581d3ac 2024-11-07T12:38:34,065 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/028767d2eb714363b3956fef5581d3ac, entries=150, sequenceid=51, filesize=30.2 K 2024-11-07T12:38:34,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/d1badeeead804229bf1ea4357cdeb68d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/d1badeeead804229bf1ea4357cdeb68d 2024-11-07T12:38:34,080 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/d1badeeead804229bf1ea4357cdeb68d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T12:38:34,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d4c272987ae44ec7b1e8e3a108692a2a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d4c272987ae44ec7b1e8e3a108692a2a 2024-11-07T12:38:34,130 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d4c272987ae44ec7b1e8e3a108692a2a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T12:38:34,137 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for bc1940ad44a4ffd987a2f6164472fdb6 in 1455ms, sequenceid=51, compaction requested=true 2024-11-07T12:38:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:34,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-07T12:38:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-07T12:38:34,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-07T12:38:34,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5090 sec 2024-11-07T12:38:34,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.5210 sec 2024-11-07T12:38:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:34,170 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:34,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-07T12:38:34,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983174171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983174177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983174180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983174183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983174183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d9fee03cf49d40678386b20543e3fc21_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983113019/Put/seqid=0 2024-11-07T12:38:34,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742427_1603 (size=14594) 2024-11-07T12:38:34,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983174287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983174287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983174297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983174298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983174298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983174497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983174497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983174507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983174508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983174508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,665 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:34,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d9fee03cf49d40678386b20543e3fc21_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d9fee03cf49d40678386b20543e3fc21_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:34,690 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ad5d02921f2e49a2890d329fa64c72d0, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:34,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ad5d02921f2e49a2890d329fa64c72d0 is 175, key is test_row_0/A:col10/1730983113019/Put/seqid=0 2024-11-07T12:38:34,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742428_1604 (size=39549) 2024-11-07T12:38:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:38:34,798 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:34,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-07T12:38:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:38:34,800 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:34,801 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:34,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983174817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983174817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983174817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983174818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983174818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:38:34,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:38:34,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:34,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:34,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:34,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:34,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:38:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:38:35,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ad5d02921f2e49a2890d329fa64c72d0 2024-11-07T12:38:35,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/737aef046f134b2eba2b23e4c5621053 is 50, key is test_row_0/B:col10/1730983113019/Put/seqid=0 2024-11-07T12:38:35,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742429_1605 (size=12001) 2024-11-07T12:38:35,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/737aef046f134b2eba2b23e4c5621053 2024-11-07T12:38:35,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/296f004b89e449dc92586b85d0ee9375 is 50, key is test_row_0/C:col10/1730983113019/Put/seqid=0 2024-11-07T12:38:35,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742430_1606 (size=12001) 2024-11-07T12:38:35,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:38:35,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:35,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983175327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983175327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983175328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:35,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:35,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983175328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983175328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:35,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:38:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:38:35,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:38:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:35,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/296f004b89e449dc92586b85d0ee9375 2024-11-07T12:38:35,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ad5d02921f2e49a2890d329fa64c72d0 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad5d02921f2e49a2890d329fa64c72d0 2024-11-07T12:38:35,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad5d02921f2e49a2890d329fa64c72d0, entries=200, sequenceid=82, filesize=38.6 K 2024-11-07T12:38:35,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/737aef046f134b2eba2b23e4c5621053 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/737aef046f134b2eba2b23e4c5621053 2024-11-07T12:38:35,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/737aef046f134b2eba2b23e4c5621053, entries=150, sequenceid=82, filesize=11.7 K 2024-11-07T12:38:35,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/296f004b89e449dc92586b85d0ee9375 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/296f004b89e449dc92586b85d0ee9375 2024-11-07T12:38:35,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/296f004b89e449dc92586b85d0ee9375, entries=150, sequenceid=82, filesize=11.7 K 2024-11-07T12:38:35,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for bc1940ad44a4ffd987a2f6164472fdb6 in 1496ms, sequenceid=82, compaction requested=true 2024-11-07T12:38:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:35,666 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:38:35,666 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:35,668 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:35,668 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/A is initiating minor compaction (all files) 2024-11-07T12:38:35,668 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/A in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,668 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad8ecfccd4244863962fde650bae818a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/af9b509181da4071a219ef918f82318e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/028767d2eb714363b3956fef5581d3ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad5d02921f2e49a2890d329fa64c72d0] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=129.3 K 2024-11-07T12:38:35,668 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,668 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad8ecfccd4244863962fde650bae818a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/af9b509181da4071a219ef918f82318e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/028767d2eb714363b3956fef5581d3ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad5d02921f2e49a2890d329fa64c72d0] 2024-11-07T12:38:35,669 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:35,669 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/B is initiating minor compaction (all files) 2024-11-07T12:38:35,669 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/B in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,669 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58f831156d8a458dbaa08c172add31ca, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/6828809ce7b84b9090142eabb6a75a45, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/d1badeeead804229bf1ea4357cdeb68d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/737aef046f134b2eba2b23e4c5621053] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=46.9 K 2024-11-07T12:38:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f831156d8a458dbaa08c172add31ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983110637 2024-11-07T12:38:35,670 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad8ecfccd4244863962fde650bae818a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983110637 2024-11-07T12:38:35,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 6828809ce7b84b9090142eabb6a75a45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1730983110687 2024-11-07T12:38:35,670 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting af9b509181da4071a219ef918f82318e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1730983110687 2024-11-07T12:38:35,670 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d1badeeead804229bf1ea4357cdeb68d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730983111831 2024-11-07T12:38:35,671 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 028767d2eb714363b3956fef5581d3ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730983111831 2024-11-07T12:38:35,671 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 737aef046f134b2eba2b23e4c5621053, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1730983113019 2024-11-07T12:38:35,671 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad5d02921f2e49a2890d329fa64c72d0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1730983113019 2024-11-07T12:38:35,683 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#B#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:35,684 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/4fabea028d524d26956271ebc4280f6e is 50, key is test_row_0/B:col10/1730983113019/Put/seqid=0 2024-11-07T12:38:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,693 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:38:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,728 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T12:38:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742431_1607 (size=12139) 2024-11-07T12:38:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,738 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107c5683dbe15be4d969f0c85207c54533c_bc1940ad44a4ffd987a2f6164472fdb6 store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,741 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107c5683dbe15be4d969f0c85207c54533c_bc1940ad44a4ffd987a2f6164472fdb6, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:35,741 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c5683dbe15be4d969f0c85207c54533c_bc1940ad44a4ffd987a2f6164472fdb6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e6944feaf3cf4ecd93c1a212b53bc5b5_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983114180/Put/seqid=0 2024-11-07T12:38:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742432_1608 (size=4469) 2024-11-07T12:38:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,781 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/4fabea028d524d26956271ebc4280f6e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/4fabea028d524d26956271ebc4280f6e 2024-11-07T12:38:35,784 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#A#compaction#515 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:35,785 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b208e4414a474f27ad0fa0d835163762 is 175, key is test_row_0/A:col10/1730983113019/Put/seqid=0 2024-11-07T12:38:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,795 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/B of bc1940ad44a4ffd987a2f6164472fdb6 into 4fabea028d524d26956271ebc4280f6e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:35,795 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:35,795 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/B, priority=12, startTime=1730983115666; duration=0sec 2024-11-07T12:38:35,795 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:35,795 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:B 2024-11-07T12:38:35,795 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,809 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:35,809 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/C is initiating minor compaction (all files) 2024-11-07T12:38:35,809 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/C in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:35,809 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/37f4313a45284e3580bbbab46d4828d2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d10d2628dae14bc584a9000bc3ee5293, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d4c272987ae44ec7b1e8e3a108692a2a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/296f004b89e449dc92586b85d0ee9375] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=46.9 K 2024-11-07T12:38:35,810 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f4313a45284e3580bbbab46d4828d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983110637 2024-11-07T12:38:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742433_1609 (size=9714) 2024-11-07T12:38:35,817 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d10d2628dae14bc584a9000bc3ee5293, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1730983110687 2024-11-07T12:38:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,822 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c272987ae44ec7b1e8e3a108692a2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730983111831 2024-11-07T12:38:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,824 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 296f004b89e449dc92586b85d0ee9375, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1730983113019 2024-11-07T12:38:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,827 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e6944feaf3cf4ecd93c1a212b53bc5b5_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6944feaf3cf4ecd93c1a212b53bc5b5_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b4200169c5af45e7a475215b6b2d3697, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:35,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b4200169c5af45e7a475215b6b2d3697 is 175, key is test_row_0/A:col10/1730983114180/Put/seqid=0 2024-11-07T12:38:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742434_1610 (size=31093) 2024-11-07T12:38:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,849 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b208e4414a474f27ad0fa0d835163762 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b208e4414a474f27ad0fa0d835163762 2024-11-07T12:38:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,855 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/A of bc1940ad44a4ffd987a2f6164472fdb6 into b208e4414a474f27ad0fa0d835163762(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:35,855 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:35,855 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/A, priority=12, startTime=1730983115666; duration=0sec 2024-11-07T12:38:35,855 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:35,855 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:A 2024-11-07T12:38:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,876 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#C#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,877 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/5b6c380677524fd3ba1fd5b04bd3cad9 is 50, key is test_row_0/C:col10/1730983113019/Put/seqid=0 2024-11-07T12:38:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742435_1611 (size=22361) 2024-11-07T12:38:35,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742436_1612 (size=12139) 2024-11-07T12:38:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,923 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/5b6c380677524fd3ba1fd5b04bd3cad9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/5b6c380677524fd3ba1fd5b04bd3cad9 2024-11-07T12:38:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,933 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/C of bc1940ad44a4ffd987a2f6164472fdb6 into 5b6c380677524fd3ba1fd5b04bd3cad9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:35,933 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:35,933 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/C, priority=12, startTime=1730983115666; duration=0sec 2024-11-07T12:38:35,933 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:35,933 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:C 2024-11-07T12:38:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:38:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,316 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b4200169c5af45e7a475215b6b2d3697 2024-11-07T12:38:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/eb56071977aa4a7b9fc86c5d7ddecf7c is 50, key is test_row_0/B:col10/1730983114180/Put/seqid=0 2024-11-07T12:38:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742437_1613 (size=9657) 2024-11-07T12:38:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:36,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,369 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/eb56071977aa4a7b9fc86c5d7ddecf7c 2024-11-07T12:38:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:36,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/83698e832cc04a408efef182aa3832ae is 50, key is test_row_0/C:col10/1730983114180/Put/seqid=0 2024-11-07T12:38:36,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742438_1614 (size=9657) 2024-11-07T12:38:36,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983176414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983176417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983176418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983176416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983176423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983176527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983176528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983176538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983176538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983176538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983176737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983176738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983176748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983176748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983176749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:36,821 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/83698e832cc04a408efef182aa3832ae 2024-11-07T12:38:36,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b4200169c5af45e7a475215b6b2d3697 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b4200169c5af45e7a475215b6b2d3697 2024-11-07T12:38:36,889 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b4200169c5af45e7a475215b6b2d3697, entries=100, sequenceid=88, filesize=21.8 K 2024-11-07T12:38:36,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/eb56071977aa4a7b9fc86c5d7ddecf7c as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/eb56071977aa4a7b9fc86c5d7ddecf7c 2024-11-07T12:38:36,938 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/eb56071977aa4a7b9fc86c5d7ddecf7c, entries=100, sequenceid=88, filesize=9.4 K 2024-11-07T12:38:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/83698e832cc04a408efef182aa3832ae as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/83698e832cc04a408efef182aa3832ae 2024-11-07T12:38:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:38:36,985 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/83698e832cc04a408efef182aa3832ae, entries=100, sequenceid=88, filesize=9.4 K 2024-11-07T12:38:36,987 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for bc1940ad44a4ffd987a2f6164472fdb6 in 1260ms, sequenceid=88, compaction requested=false 2024-11-07T12:38:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-07T12:38:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-07T12:38:37,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-07T12:38:37,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1920 sec 2024-11-07T12:38:37,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.2050 sec 2024-11-07T12:38:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:37,051 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:37,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-07T12:38:37,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983177058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983177058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983177058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983177070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983177070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079fe8918552804164a88313351305e3c1_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983117049/Put/seqid=0 2024-11-07T12:38:37,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742439_1615 (size=17034) 2024-11-07T12:38:37,137 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,148 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079fe8918552804164a88313351305e3c1_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079fe8918552804164a88313351305e3c1_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:37,150 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f5c692cb9132414084ff8807ca51f81b, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:37,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f5c692cb9132414084ff8807ca51f81b is 175, key is test_row_0/A:col10/1730983117049/Put/seqid=0 2024-11-07T12:38:37,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983177169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983177181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742440_1616 (size=48139) 2024-11-07T12:38:37,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f5c692cb9132414084ff8807ca51f81b 2024-11-07T12:38:37,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/a969ef64a6024803b5de85c6b64001db is 50, key is test_row_0/B:col10/1730983117049/Put/seqid=0 2024-11-07T12:38:37,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742441_1617 (size=12001) 2024-11-07T12:38:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983177389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983177393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983177567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/a969ef64a6024803b5de85c6b64001db 2024-11-07T12:38:37,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/2d752af9b2fe46d1a88adabea361616b is 50, key is test_row_0/C:col10/1730983117049/Put/seqid=0 2024-11-07T12:38:37,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742442_1618 (size=12001) 2024-11-07T12:38:37,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/2d752af9b2fe46d1a88adabea361616b 2024-11-07T12:38:37,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f5c692cb9132414084ff8807ca51f81b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f5c692cb9132414084ff8807ca51f81b 2024-11-07T12:38:37,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f5c692cb9132414084ff8807ca51f81b, entries=250, sequenceid=123, filesize=47.0 K 2024-11-07T12:38:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/a969ef64a6024803b5de85c6b64001db as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a969ef64a6024803b5de85c6b64001db 2024-11-07T12:38:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a969ef64a6024803b5de85c6b64001db, entries=150, sequenceid=123, filesize=11.7 K 2024-11-07T12:38:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/2d752af9b2fe46d1a88adabea361616b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/2d752af9b2fe46d1a88adabea361616b 2024-11-07T12:38:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/2d752af9b2fe46d1a88adabea361616b, entries=150, sequenceid=123, filesize=11.7 K 2024-11-07T12:38:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for bc1940ad44a4ffd987a2f6164472fdb6 in 625ms, sequenceid=123, compaction requested=true 2024-11-07T12:38:37,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:37,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:37,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:37,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:37,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:37,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:38:37,676 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,677 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,678 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:37,678 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/A is initiating minor compaction (all files) 2024-11-07T12:38:37,678 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/A in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:37,678 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b208e4414a474f27ad0fa0d835163762, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b4200169c5af45e7a475215b6b2d3697, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f5c692cb9132414084ff8807ca51f81b] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=99.2 K 2024-11-07T12:38:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,678 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:37,678 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b208e4414a474f27ad0fa0d835163762, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b4200169c5af45e7a475215b6b2d3697, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f5c692cb9132414084ff8807ca51f81b] 2024-11-07T12:38:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,678 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:37,679 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/B is initiating minor compaction (all files) 2024-11-07T12:38:37,679 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b208e4414a474f27ad0fa0d835163762, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1730983113019 2024-11-07T12:38:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,679 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/B in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:37,679 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/4fabea028d524d26956271ebc4280f6e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/eb56071977aa4a7b9fc86c5d7ddecf7c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a969ef64a6024803b5de85c6b64001db] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=33.0 K 2024-11-07T12:38:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,679 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fabea028d524d26956271ebc4280f6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1730983113019 2024-11-07T12:38:37,679 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4200169c5af45e7a475215b6b2d3697, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730983114176 2024-11-07T12:38:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,679 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting eb56071977aa4a7b9fc86c5d7ddecf7c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730983114176 2024-11-07T12:38:37,679 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c692cb9132414084ff8807ca51f81b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983116412 2024-11-07T12:38:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,680 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a969ef64a6024803b5de85c6b64001db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983116412 2024-11-07T12:38:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,687 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,688 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#B#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,688 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/5a66f1ded79a4e98b5b03f0fc981aa8a is 50, key is test_row_0/B:col10/1730983117049/Put/seqid=0 2024-11-07T12:38:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,691 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411070e5cf3b6a406434699898de30c676d55_bc1940ad44a4ffd987a2f6164472fdb6 store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,693 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411070e5cf3b6a406434699898de30c676d55_bc1940ad44a4ffd987a2f6164472fdb6, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,693 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070e5cf3b6a406434699898de30c676d55_bc1940ad44a4ffd987a2f6164472fdb6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742443_1619 (size=12241) 2024-11-07T12:38:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742444_1620 (size=4469) 2024-11-07T12:38:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,702 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/5a66f1ded79a4e98b5b03f0fc981aa8a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5a66f1ded79a4e98b5b03f0fc981aa8a 2024-11-07T12:38:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,709 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#A#compaction#523 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:37,710 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/94914825e5b24523a72a3f5d393c1c74 is 175, key is test_row_0/A:col10/1730983117049/Put/seqid=0 2024-11-07T12:38:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742445_1621 (size=31195) 2024-11-07T12:38:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:37,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:37,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:38:37,716 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/B of bc1940ad44a4ffd987a2f6164472fdb6 into 5a66f1ded79a4e98b5b03f0fc981aa8a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:37,716 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:37,716 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/B, priority=13, startTime=1730983117676; duration=0sec 2024-11-07T12:38:37,716 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:37,716 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:B 2024-11-07T12:38:37,716 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:37,719 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:37,719 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/C is initiating minor compaction (all files) 2024-11-07T12:38:37,719 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/C in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:37,719 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/5b6c380677524fd3ba1fd5b04bd3cad9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/83698e832cc04a408efef182aa3832ae, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/2d752af9b2fe46d1a88adabea361616b] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=33.0 K 2024-11-07T12:38:37,720 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b6c380677524fd3ba1fd5b04bd3cad9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1730983113019 2024-11-07T12:38:37,720 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 83698e832cc04a408efef182aa3832ae, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730983114176 2024-11-07T12:38:37,721 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d752af9b2fe46d1a88adabea361616b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983116412 2024-11-07T12:38:37,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073421476c48c643afaa380a128c00fabd_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983117713/Put/seqid=0 2024-11-07T12:38:37,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742446_1622 (size=17234) 2024-11-07T12:38:37,734 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#C#compaction#526 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:37,734 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/c514e949b5dc4901b2e39dae16b04a24 is 50, key is test_row_0/C:col10/1730983117049/Put/seqid=0 2024-11-07T12:38:37,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742447_1623 (size=12241) 2024-11-07T12:38:37,749 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/c514e949b5dc4901b2e39dae16b04a24 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c514e949b5dc4901b2e39dae16b04a24 2024-11-07T12:38:37,753 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/C of bc1940ad44a4ffd987a2f6164472fdb6 into c514e949b5dc4901b2e39dae16b04a24(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:37,753 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:37,753 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/C, priority=13, startTime=1730983117676; duration=0sec 2024-11-07T12:38:37,753 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:37,753 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:C 2024-11-07T12:38:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983177809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983177810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983177929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983177929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:37,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,120 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/94914825e5b24523a72a3f5d393c1c74 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/94914825e5b24523a72a3f5d393c1c74 2024-11-07T12:38:38,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:38,131 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/A of bc1940ad44a4ffd987a2f6164472fdb6 into 94914825e5b24523a72a3f5d393c1c74(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:38,131 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:38,131 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/A, priority=13, startTime=1730983117676; duration=0sec 2024-11-07T12:38:38,131 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:38,131 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:A 2024-11-07T12:38:38,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073421476c48c643afaa380a128c00fabd_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073421476c48c643afaa380a128c00fabd_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:38,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/c8894980c2fb4b4183e43ef8f33342a1, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:38,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/c8894980c2fb4b4183e43ef8f33342a1 is 175, key is test_row_0/A:col10/1730983117713/Put/seqid=0 2024-11-07T12:38:38,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983178147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983178148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742448_1624 (size=48339) 2024-11-07T12:38:38,212 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/c8894980c2fb4b4183e43ef8f33342a1 2024-11-07T12:38:38,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/38c25c1535fb4ae59f6890097d820202 is 50, key is test_row_0/B:col10/1730983117713/Put/seqid=0 2024-11-07T12:38:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742449_1625 (size=12101) 2024-11-07T12:38:38,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983178466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983178461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983178577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,579 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in bc1940ad44a4ffd987a2f6164472fdb6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:38,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983178587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983178588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,592 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in bc1940ad44a4ffd987a2f6164472fdb6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:38,592 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_2' in bc1940ad44a4ffd987a2f6164472fdb6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/38c25c1535fb4ae59f6890097d820202 2024-11-07T12:38:38,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/f27d08f094b54636a3a067a20d6c206f is 50, key is test_row_0/C:col10/1730983117713/Put/seqid=0 2024-11-07T12:38:38,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742450_1626 (size=12101) 2024-11-07T12:38:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:38:38,978 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:38,979 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-07T12:38:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,982 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983178977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983178978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,983 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:38,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:38:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:38:39,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T12:38:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:39,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:39,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:39,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/f27d08f094b54636a3a067a20d6c206f 2024-11-07T12:38:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/c8894980c2fb4b4183e43ef8f33342a1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/c8894980c2fb4b4183e43ef8f33342a1 2024-11-07T12:38:39,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/c8894980c2fb4b4183e43ef8f33342a1, entries=250, sequenceid=134, filesize=47.2 K 2024-11-07T12:38:39,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/38c25c1535fb4ae59f6890097d820202 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/38c25c1535fb4ae59f6890097d820202 2024-11-07T12:38:39,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/38c25c1535fb4ae59f6890097d820202, entries=150, sequenceid=134, filesize=11.8 K 2024-11-07T12:38:39,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/f27d08f094b54636a3a067a20d6c206f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/f27d08f094b54636a3a067a20d6c206f 2024-11-07T12:38:39,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/f27d08f094b54636a3a067a20d6c206f, entries=150, sequenceid=134, filesize=11.8 K 2024-11-07T12:38:39,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for bc1940ad44a4ffd987a2f6164472fdb6 in 1445ms, sequenceid=134, compaction requested=false 2024-11-07T12:38:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:39,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T12:38:39,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:39,289 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:38:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:38:39,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078a50340e0258447bbff1abbb01f85f8a_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983117808/Put/seqid=0 2024-11-07T12:38:39,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742451_1627 (size=12304) 2024-11-07T12:38:39,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,340 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078a50340e0258447bbff1abbb01f85f8a_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a50340e0258447bbff1abbb01f85f8a_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:39,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f6cc518e87c04b2386a3583de90e19be, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:39,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f6cc518e87c04b2386a3583de90e19be is 175, key is test_row_0/A:col10/1730983117808/Put/seqid=0 2024-11-07T12:38:39,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742452_1628 (size=31105) 2024-11-07T12:38:39,359 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f6cc518e87c04b2386a3583de90e19be 2024-11-07T12:38:39,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/5da5331ed82f4769b0c16541c3d91e8d is 50, key is test_row_0/B:col10/1730983117808/Put/seqid=0 2024-11-07T12:38:39,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742453_1629 (size=12151) 2024-11-07T12:38:39,403 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/5da5331ed82f4769b0c16541c3d91e8d 2024-11-07T12:38:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/59ae84977453405b89400a35e0efd1f7 is 50, key is test_row_0/C:col10/1730983117808/Put/seqid=0 2024-11-07T12:38:39,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742454_1630 (size=12151) 2024-11-07T12:38:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:38:39,881 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/59ae84977453405b89400a35e0efd1f7 2024-11-07T12:38:39,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f6cc518e87c04b2386a3583de90e19be as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f6cc518e87c04b2386a3583de90e19be 2024-11-07T12:38:39,970 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f6cc518e87c04b2386a3583de90e19be, entries=150, sequenceid=163, filesize=30.4 K 2024-11-07T12:38:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/5da5331ed82f4769b0c16541c3d91e8d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5da5331ed82f4769b0c16541c3d91e8d 2024-11-07T12:38:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,005 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5da5331ed82f4769b0c16541c3d91e8d, entries=150, sequenceid=163, filesize=11.9 K 2024-11-07T12:38:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/59ae84977453405b89400a35e0efd1f7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59ae84977453405b89400a35e0efd1f7 2024-11-07T12:38:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,029 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59ae84977453405b89400a35e0efd1f7, entries=150, sequenceid=163, filesize=11.9 K 2024-11-07T12:38:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,041 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for bc1940ad44a4ffd987a2f6164472fdb6 in 752ms, sequenceid=163, compaction requested=true 2024-11-07T12:38:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-07T12:38:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-07T12:38:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-07T12:38:40,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-11-07T12:38:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.0700 sec 2024-11-07T12:38:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:38:40,127 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-07T12:38:40,132 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:38:40,134 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:40,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:40,163 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:40,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:38:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072fdeaf333bfc41f981baa47fb34e5833_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983120154/Put/seqid=0 2024-11-07T12:38:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:38:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742456_1632 (size=24758) 2024-11-07T12:38:40,260 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,280 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072fdeaf333bfc41f981baa47fb34e5833_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072fdeaf333bfc41f981baa47fb34e5833_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:40,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/19f2f9e43c584e4d875594bdb9735b0b, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/19f2f9e43c584e4d875594bdb9735b0b is 175, key is test_row_0/A:col10/1730983120154/Put/seqid=0 2024-11-07T12:38:40,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742455_1631 (size=74394) 2024-11-07T12:38:40,301 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/19f2f9e43c584e4d875594bdb9735b0b 2024-11-07T12:38:40,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/bb9f94ce3fa341228182ef21a78855c9 is 50, key is test_row_0/B:col10/1730983120154/Put/seqid=0 2024-11-07T12:38:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983180350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983180363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742457_1633 (size=12151) 2024-11-07T12:38:40,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/bb9f94ce3fa341228182ef21a78855c9 2024-11-07T12:38:40,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/59c000a9b0a34a8c9164e44e1da3a1e3 is 50, key is test_row_0/C:col10/1730983120154/Put/seqid=0 2024-11-07T12:38:40,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742458_1634 (size=12151) 2024-11-07T12:38:40,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/59c000a9b0a34a8c9164e44e1da3a1e3 2024-11-07T12:38:40,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:40,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:40,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:38:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983180473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/19f2f9e43c584e4d875594bdb9735b0b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/19f2f9e43c584e4d875594bdb9735b0b 2024-11-07T12:38:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983180479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/19f2f9e43c584e4d875594bdb9735b0b, entries=400, sequenceid=174, filesize=72.7 K 2024-11-07T12:38:40,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/bb9f94ce3fa341228182ef21a78855c9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bb9f94ce3fa341228182ef21a78855c9 2024-11-07T12:38:40,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bb9f94ce3fa341228182ef21a78855c9, entries=150, sequenceid=174, filesize=11.9 K 2024-11-07T12:38:40,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/59c000a9b0a34a8c9164e44e1da3a1e3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59c000a9b0a34a8c9164e44e1da3a1e3 2024-11-07T12:38:40,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59c000a9b0a34a8c9164e44e1da3a1e3, entries=150, sequenceid=174, filesize=11.9 K 2024-11-07T12:38:40,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for bc1940ad44a4ffd987a2f6164472fdb6 in 366ms, sequenceid=174, compaction requested=true 2024-11-07T12:38:40,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:40,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:40,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:40,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:40,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:38:40,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:40,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:38:40,530 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:40,532 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:40,540 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 185033 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:40,541 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/A is initiating minor compaction (all files) 2024-11-07T12:38:40,541 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/A in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,541 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/94914825e5b24523a72a3f5d393c1c74, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/c8894980c2fb4b4183e43ef8f33342a1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f6cc518e87c04b2386a3583de90e19be, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/19f2f9e43c584e4d875594bdb9735b0b] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=180.7 K 2024-11-07T12:38:40,541 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,548 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/94914825e5b24523a72a3f5d393c1c74, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/c8894980c2fb4b4183e43ef8f33342a1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f6cc518e87c04b2386a3583de90e19be, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/19f2f9e43c584e4d875594bdb9735b0b] 2024-11-07T12:38:40,552 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:40,552 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/C is initiating minor compaction (all files) 2024-11-07T12:38:40,553 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/C in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,553 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c514e949b5dc4901b2e39dae16b04a24, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/f27d08f094b54636a3a067a20d6c206f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59ae84977453405b89400a35e0efd1f7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59c000a9b0a34a8c9164e44e1da3a1e3] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=47.5 K 2024-11-07T12:38:40,557 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94914825e5b24523a72a3f5d393c1c74, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983116412 2024-11-07T12:38:40,560 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting c514e949b5dc4901b2e39dae16b04a24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983116412 2024-11-07T12:38:40,565 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8894980c2fb4b4183e43ef8f33342a1, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1730983117055 2024-11-07T12:38:40,566 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting f27d08f094b54636a3a067a20d6c206f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1730983117055 2024-11-07T12:38:40,569 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6cc518e87c04b2386a3583de90e19be, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1730983117765 2024-11-07T12:38:40,572 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 59ae84977453405b89400a35e0efd1f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1730983117765 2024-11-07T12:38:40,572 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f2f9e43c584e4d875594bdb9735b0b, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983120045 2024-11-07T12:38:40,580 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c000a9b0a34a8c9164e44e1da3a1e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983120121 2024-11-07T12:38:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:40,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:40,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:38:40,613 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:40,617 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107243b00e5d0ae496daee51283ce8dbd54_bc1940ad44a4ffd987a2f6164472fdb6 store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:40,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:40,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:40,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,620 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107243b00e5d0ae496daee51283ce8dbd54_bc1940ad44a4ffd987a2f6164472fdb6, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:40,620 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107243b00e5d0ae496daee51283ce8dbd54_bc1940ad44a4ffd987a2f6164472fdb6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,639 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#C#compaction#536 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:40,639 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/020c4700d3f34d6b86eb0e9c70d34ada is 50, key is test_row_0/C:col10/1730983120154/Put/seqid=0 2024-11-07T12:38:40,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071d1a8e1669644aa7a1020330c53ca44d_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983120343/Put/seqid=0 2024-11-07T12:38:40,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742459_1635 (size=4469) 2024-11-07T12:38:40,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983180658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983180661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983180670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742461_1637 (size=14794) 2024-11-07T12:38:40,686 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:40,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983180687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,709 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071d1a8e1669644aa7a1020330c53ca44d_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071d1a8e1669644aa7a1020330c53ca44d_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:40,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742460_1636 (size=12527) 2024-11-07T12:38:40,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983180707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,716 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f7c7dabb0b784743b64d45e1e439eb97, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:40,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f7c7dabb0b784743b64d45e1e439eb97 is 175, key is test_row_0/A:col10/1730983120343/Put/seqid=0 2024-11-07T12:38:40,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:38:40,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742462_1638 (size=39749) 2024-11-07T12:38:40,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:40,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983180781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983180781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983180789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:40,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:40,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:40,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:40,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983181001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983181001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983181008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983181009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983181021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,068 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#A#compaction#535 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:41,068 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/3aeb15c232c64c3aa777a153cf74b22f is 175, key is test_row_0/A:col10/1730983120154/Put/seqid=0 2024-11-07T12:38:41,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:41,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:41,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:41,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:41,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742463_1639 (size=31481) 2024-11-07T12:38:41,129 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/3aeb15c232c64c3aa777a153cf74b22f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/3aeb15c232c64c3aa777a153cf74b22f 2024-11-07T12:38:41,141 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/020c4700d3f34d6b86eb0e9c70d34ada as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/020c4700d3f34d6b86eb0e9c70d34ada 2024-11-07T12:38:41,143 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/A of bc1940ad44a4ffd987a2f6164472fdb6 into 3aeb15c232c64c3aa777a153cf74b22f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:41,143 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:41,143 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/A, priority=12, startTime=1730983120530; duration=0sec 2024-11-07T12:38:41,143 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:41,143 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:A 2024-11-07T12:38:41,143 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:41,149 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:41,149 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/B is initiating minor compaction (all files) 2024-11-07T12:38:41,149 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/B in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:41,149 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5a66f1ded79a4e98b5b03f0fc981aa8a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/38c25c1535fb4ae59f6890097d820202, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5da5331ed82f4769b0c16541c3d91e8d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bb9f94ce3fa341228182ef21a78855c9] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=47.5 K 2024-11-07T12:38:41,149 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a66f1ded79a4e98b5b03f0fc981aa8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983116412 2024-11-07T12:38:41,150 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c25c1535fb4ae59f6890097d820202, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1730983117055 2024-11-07T12:38:41,150 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da5331ed82f4769b0c16541c3d91e8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1730983117765 2024-11-07T12:38:41,151 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb9f94ce3fa341228182ef21a78855c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983120121 2024-11-07T12:38:41,153 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/C of bc1940ad44a4ffd987a2f6164472fdb6 into 020c4700d3f34d6b86eb0e9c70d34ada(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:41,153 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:41,153 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/C, priority=12, startTime=1730983120530; duration=0sec 2024-11-07T12:38:41,153 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:41,153 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:C 2024-11-07T12:38:41,179 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#B#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:41,179 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/71cf2ea4b74345dd8f159c2d5f14362e is 50, key is test_row_0/B:col10/1730983120154/Put/seqid=0 2024-11-07T12:38:41,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f7c7dabb0b784743b64d45e1e439eb97 2024-11-07T12:38:41,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742464_1640 (size=12527) 2024-11-07T12:38:41,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/8266ef85189a46e49e5db6a275a9d282 is 50, key is test_row_0/B:col10/1730983120343/Put/seqid=0 2024-11-07T12:38:41,242 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/71cf2ea4b74345dd8f159c2d5f14362e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/71cf2ea4b74345dd8f159c2d5f14362e 2024-11-07T12:38:41,249 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/B of bc1940ad44a4ffd987a2f6164472fdb6 into 71cf2ea4b74345dd8f159c2d5f14362e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:41,249 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:41,249 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/B, priority=12, startTime=1730983120530; duration=0sec 2024-11-07T12:38:41,250 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:41,250 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:B 2024-11-07T12:38:41,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:41,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742465_1641 (size=12151) 2024-11-07T12:38:41,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:41,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:41,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:41,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/8266ef85189a46e49e5db6a275a9d282 2024-11-07T12:38:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:38:41,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983181317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983181318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983181329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/8ecd745d04c7438bbc04e9b437cd19d6 is 50, key is test_row_0/C:col10/1730983120343/Put/seqid=0 2024-11-07T12:38:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742466_1642 (size=12151) 2024-11-07T12:38:41,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/8ecd745d04c7438bbc04e9b437cd19d6 2024-11-07T12:38:41,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:41,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:41,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:41,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:41,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:41,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/f7c7dabb0b784743b64d45e1e439eb97 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f7c7dabb0b784743b64d45e1e439eb97 2024-11-07T12:38:41,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f7c7dabb0b784743b64d45e1e439eb97, entries=200, sequenceid=200, filesize=38.8 K 2024-11-07T12:38:41,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/8266ef85189a46e49e5db6a275a9d282 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8266ef85189a46e49e5db6a275a9d282 2024-11-07T12:38:41,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8266ef85189a46e49e5db6a275a9d282, entries=150, sequenceid=200, filesize=11.9 K 2024-11-07T12:38:41,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/8ecd745d04c7438bbc04e9b437cd19d6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/8ecd745d04c7438bbc04e9b437cd19d6 2024-11-07T12:38:41,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/8ecd745d04c7438bbc04e9b437cd19d6, entries=150, sequenceid=200, filesize=11.9 K 2024-11-07T12:38:41,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for bc1940ad44a4ffd987a2f6164472fdb6 in 926ms, sequenceid=200, compaction requested=false 2024-11-07T12:38:41,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:41,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:38:41,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:41,581 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T12:38:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:41,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:41,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110788d7ffc5182b4dd7884307bad347fb21_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983121571/Put/seqid=0 2024-11-07T12:38:41,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742467_1643 (size=17284) 2024-11-07T12:38:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:41,677 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110788d7ffc5182b4dd7884307bad347fb21_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110788d7ffc5182b4dd7884307bad347fb21_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:41,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/7bb0c062572641eca8073fc47c8b9136, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:41,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/7bb0c062572641eca8073fc47c8b9136 is 175, key is test_row_0/A:col10/1730983121571/Put/seqid=0 2024-11-07T12:38:41,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742468_1644 (size=48389) 2024-11-07T12:38:41,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983181726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983181730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983181838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983181838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983181841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983181851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:41,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983181853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983182047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983182068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,121 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/7bb0c062572641eca8073fc47c8b9136 2024-11-07T12:38:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/fbd2aba7c6db47cba023cf53b527af42 is 50, key is test_row_0/B:col10/1730983121571/Put/seqid=0 2024-11-07T12:38:42,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742469_1645 (size=12151) 2024-11-07T12:38:42,222 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/fbd2aba7c6db47cba023cf53b527af42 2024-11-07T12:38:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/72c79940bb4048f999a790800585c3ac is 50, key is test_row_0/C:col10/1730983121571/Put/seqid=0 2024-11-07T12:38:42,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742470_1646 (size=12151) 2024-11-07T12:38:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:38:42,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983182357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983182377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,677 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/72c79940bb4048f999a790800585c3ac 2024-11-07T12:38:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/7bb0c062572641eca8073fc47c8b9136 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bb0c062572641eca8073fc47c8b9136 2024-11-07T12:38:42,696 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bb0c062572641eca8073fc47c8b9136, entries=250, sequenceid=215, filesize=47.3 K 2024-11-07T12:38:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/fbd2aba7c6db47cba023cf53b527af42 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/fbd2aba7c6db47cba023cf53b527af42 2024-11-07T12:38:42,702 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/fbd2aba7c6db47cba023cf53b527af42, entries=150, sequenceid=215, filesize=11.9 K 2024-11-07T12:38:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/72c79940bb4048f999a790800585c3ac as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/72c79940bb4048f999a790800585c3ac 2024-11-07T12:38:42,715 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/72c79940bb4048f999a790800585c3ac, entries=150, sequenceid=215, filesize=11.9 K 2024-11-07T12:38:42,715 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bc1940ad44a4ffd987a2f6164472fdb6 in 1135ms, sequenceid=215, compaction requested=true 2024-11-07T12:38:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-07T12:38:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-07T12:38:42,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-07T12:38:42,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5890 sec 2024-11-07T12:38:42,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.6290 sec 2024-11-07T12:38:42,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:42,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:38:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:42,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110735fc2621a4b5402e8d4b09e57d10abf7_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983121707/Put/seqid=0 2024-11-07T12:38:42,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742471_1647 (size=14794) 2024-11-07T12:38:42,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:42,891 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110735fc2621a4b5402e8d4b09e57d10abf7_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735fc2621a4b5402e8d4b09e57d10abf7_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:42,892 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/9d067c0b7d084bb3b983e9a9b2b4d93f, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:42,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/9d067c0b7d084bb3b983e9a9b2b4d93f is 175, key is test_row_0/A:col10/1730983121707/Put/seqid=0 2024-11-07T12:38:42,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983182887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983182887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983182890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983182895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983182896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:42,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742472_1648 (size=39749) 2024-11-07T12:38:42,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/9d067c0b7d084bb3b983e9a9b2b4d93f 2024-11-07T12:38:42,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/58430af085d44f9e87d1a337aa750b97 is 50, key is test_row_0/B:col10/1730983121707/Put/seqid=0 2024-11-07T12:38:42,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742473_1649 (size=12151) 2024-11-07T12:38:43,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983183009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983183010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983183012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983183013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983183218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983183219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983183229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983183232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/58430af085d44f9e87d1a337aa750b97 2024-11-07T12:38:43,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d74bb482f1784f0eae424de5c23f1177 is 50, key is test_row_0/C:col10/1730983121707/Put/seqid=0 2024-11-07T12:38:43,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742474_1650 (size=12151) 2024-11-07T12:38:43,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d74bb482f1784f0eae424de5c23f1177 2024-11-07T12:38:43,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/9d067c0b7d084bb3b983e9a9b2b4d93f as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/9d067c0b7d084bb3b983e9a9b2b4d93f 2024-11-07T12:38:43,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/9d067c0b7d084bb3b983e9a9b2b4d93f, entries=200, sequenceid=240, filesize=38.8 K 2024-11-07T12:38:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/58430af085d44f9e87d1a337aa750b97 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58430af085d44f9e87d1a337aa750b97 2024-11-07T12:38:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58430af085d44f9e87d1a337aa750b97, entries=150, sequenceid=240, filesize=11.9 K 2024-11-07T12:38:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d74bb482f1784f0eae424de5c23f1177 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d74bb482f1784f0eae424de5c23f1177 2024-11-07T12:38:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d74bb482f1784f0eae424de5c23f1177, entries=150, sequenceid=240, filesize=11.9 K 2024-11-07T12:38:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for bc1940ad44a4ffd987a2f6164472fdb6 in 535ms, sequenceid=240, compaction requested=true 2024-11-07T12:38:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:43,401 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:43,401 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,402 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,402 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/B is initiating minor compaction (all files) 2024-11-07T12:38:43,402 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/B in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:43,402 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/71cf2ea4b74345dd8f159c2d5f14362e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8266ef85189a46e49e5db6a275a9d282, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/fbd2aba7c6db47cba023cf53b527af42, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58430af085d44f9e87d1a337aa750b97] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=47.8 K 2024-11-07T12:38:43,402 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159368 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:43,402 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/A is initiating minor compaction (all files) 2024-11-07T12:38:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,402 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/A in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:43,402 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/3aeb15c232c64c3aa777a153cf74b22f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f7c7dabb0b784743b64d45e1e439eb97, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bb0c062572641eca8073fc47c8b9136, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/9d067c0b7d084bb3b983e9a9b2b4d93f] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=155.6 K 2024-11-07T12:38:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,402 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:43,403 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/3aeb15c232c64c3aa777a153cf74b22f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f7c7dabb0b784743b64d45e1e439eb97, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bb0c062572641eca8073fc47c8b9136, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/9d067c0b7d084bb3b983e9a9b2b4d93f] 2024-11-07T12:38:43,403 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 71cf2ea4b74345dd8f159c2d5f14362e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983120121 2024-11-07T12:38:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,403 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aeb15c232c64c3aa777a153cf74b22f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983120121 2024-11-07T12:38:43,403 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8266ef85189a46e49e5db6a275a9d282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730983120343 2024-11-07T12:38:43,403 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c7dabb0b784743b64d45e1e439eb97, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730983120343 2024-11-07T12:38:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,403 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting fbd2aba7c6db47cba023cf53b527af42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1730983120641 2024-11-07T12:38:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,404 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb0c062572641eca8073fc47c8b9136, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1730983120641 2024-11-07T12:38:43,404 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 58430af085d44f9e87d1a337aa750b97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730983121707 2024-11-07T12:38:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,404 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d067c0b7d084bb3b983e9a9b2b4d93f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730983121632 2024-11-07T12:38:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,415 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#B#compaction#547 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,416 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/8acb4c7b4b884121b1ea881a661cf99e is 50, key is test_row_0/B:col10/1730983121707/Put/seqid=0 2024-11-07T12:38:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,417 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,420 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110760a52240cfd34607a6dc4c25098cba58_bc1940ad44a4ffd987a2f6164472fdb6 store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,422 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110760a52240cfd34607a6dc4c25098cba58_bc1940ad44a4ffd987a2f6164472fdb6, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:43,422 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110760a52240cfd34607a6dc4c25098cba58_bc1940ad44a4ffd987a2f6164472fdb6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742475_1651 (size=12663) 2024-11-07T12:38:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742476_1652 (size=4469) 2024-11-07T12:38:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,432 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#A#compaction#548 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,433 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/2eaf969856824c039259db60436e9f46 is 175, key is test_row_0/A:col10/1730983121707/Put/seqid=0 2024-11-07T12:38:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742477_1653 (size=31617) 2024-11-07T12:38:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:43,539 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:43,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T12:38:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070f8fbfca66e64f3eb629508180ec4094_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_1/A:col10/1730983123533/Put/seqid=0 2024-11-07T12:38:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742478_1654 (size=14794) 2024-11-07T12:38:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983183595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983183601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983183603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983183605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983183710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983183719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983183718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983183719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,839 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/8acb4c7b4b884121b1ea881a661cf99e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8acb4c7b4b884121b1ea881a661cf99e 2024-11-07T12:38:43,854 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/B of bc1940ad44a4ffd987a2f6164472fdb6 into 8acb4c7b4b884121b1ea881a661cf99e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:43,854 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:43,854 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/B, priority=12, startTime=1730983123401; duration=0sec 2024-11-07T12:38:43,854 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:43,854 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:B 2024-11-07T12:38:43,854 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:38:43,855 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:38:43,855 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/C is initiating minor compaction (all files) 2024-11-07T12:38:43,855 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/C in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:43,855 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/020c4700d3f34d6b86eb0e9c70d34ada, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/8ecd745d04c7438bbc04e9b437cd19d6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/72c79940bb4048f999a790800585c3ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d74bb482f1784f0eae424de5c23f1177] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=47.8 K 2024-11-07T12:38:43,855 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 020c4700d3f34d6b86eb0e9c70d34ada, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983120121 2024-11-07T12:38:43,856 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ecd745d04c7438bbc04e9b437cd19d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730983120343 2024-11-07T12:38:43,856 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c79940bb4048f999a790800585c3ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1730983120641 2024-11-07T12:38:43,856 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d74bb482f1784f0eae424de5c23f1177, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730983121707 2024-11-07T12:38:43,864 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/2eaf969856824c039259db60436e9f46 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2eaf969856824c039259db60436e9f46 2024-11-07T12:38:43,879 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/A of bc1940ad44a4ffd987a2f6164472fdb6 into 2eaf969856824c039259db60436e9f46(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:43,879 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:43,879 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/A, priority=12, startTime=1730983123400; duration=0sec 2024-11-07T12:38:43,879 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:43,879 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:A 2024-11-07T12:38:43,880 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#C#compaction#550 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:43,881 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/90727b80dc274242aaa860780be8d049 is 50, key is test_row_0/C:col10/1730983121707/Put/seqid=0 2024-11-07T12:38:43,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983183913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983183917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742479_1655 (size=12663) 2024-11-07T12:38:43,925 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bc1940ad44a4ffd987a2f6164472fdb6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983183932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983183932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983183933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:43,979 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:43,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070f8fbfca66e64f3eb629508180ec4094_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f8fbfca66e64f3eb629508180ec4094_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:43,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/53a7646b24fc4396b59535097f9b7a12, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:43,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/53a7646b24fc4396b59535097f9b7a12 is 175, key is test_row_1/A:col10/1730983123533/Put/seqid=0 2024-11-07T12:38:44,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742480_1656 (size=39745) 2024-11-07T12:38:44,036 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/53a7646b24fc4396b59535097f9b7a12 2024-11-07T12:38:44,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/24eb5a1e686a45e2abf3430ff4d2962d is 50, key is test_row_1/B:col10/1730983123533/Put/seqid=0 2024-11-07T12:38:44,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742481_1657 (size=9757) 2024-11-07T12:38:44,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983184228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983184237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983184237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983184247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:38:44,308 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:44,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-07T12:38:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:38:44,311 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:44,312 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:44,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:44,326 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/90727b80dc274242aaa860780be8d049 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/90727b80dc274242aaa860780be8d049 2024-11-07T12:38:44,330 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/C of bc1940ad44a4ffd987a2f6164472fdb6 into 90727b80dc274242aaa860780be8d049(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:44,331 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:44,331 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/C, priority=12, startTime=1730983123401; duration=0sec 2024-11-07T12:38:44,331 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:44,331 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:C 2024-11-07T12:38:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:38:44,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:44,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/24eb5a1e686a45e2abf3430ff4d2962d 2024-11-07T12:38:44,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/42025a0f3cbd4a418902d09575a4eb38 is 50, key is test_row_1/C:col10/1730983123533/Put/seqid=0 2024-11-07T12:38:44,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742482_1658 (size=9757) 2024-11-07T12:38:44,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/42025a0f3cbd4a418902d09575a4eb38 2024-11-07T12:38:44,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/53a7646b24fc4396b59535097f9b7a12 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/53a7646b24fc4396b59535097f9b7a12 2024-11-07T12:38:44,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/53a7646b24fc4396b59535097f9b7a12, entries=200, sequenceid=254, filesize=38.8 K 2024-11-07T12:38:44,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/24eb5a1e686a45e2abf3430ff4d2962d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/24eb5a1e686a45e2abf3430ff4d2962d 2024-11-07T12:38:44,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/24eb5a1e686a45e2abf3430ff4d2962d, entries=100, sequenceid=254, filesize=9.5 K 2024-11-07T12:38:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/42025a0f3cbd4a418902d09575a4eb38 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/42025a0f3cbd4a418902d09575a4eb38 2024-11-07T12:38:44,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:44,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:38:44,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/42025a0f3cbd4a418902d09575a4eb38, entries=100, sequenceid=254, filesize=9.5 K 2024-11-07T12:38:44,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for bc1940ad44a4ffd987a2f6164472fdb6 in 1106ms, sequenceid=254, compaction requested=false 2024-11-07T12:38:44,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:44,756 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:44,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:38:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:44,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983184782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983184785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:44,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983184792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983184793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107959d5dd0b8c643ae8ac3d1da2383cf7e_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983123599/Put/seqid=0 2024-11-07T12:38:44,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742483_1659 (size=14994) 2024-11-07T12:38:44,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983184901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983184903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983184909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983184915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:38:44,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:44,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:44,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:45,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983185121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983185121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983185124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:45,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983185127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:45,273 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107959d5dd0b8c643ae8ac3d1da2383cf7e_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107959d5dd0b8c643ae8ac3d1da2383cf7e_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:45,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/008d9283618849f08044c0cf76f77e20, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:45,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/008d9283618849f08044c0cf76f77e20 is 175, key is test_row_0/A:col10/1730983123599/Put/seqid=0 2024-11-07T12:38:45,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:45,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:45,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742484_1660 (size=39949) 2024-11-07T12:38:45,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983185437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983185438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983185441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:38:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983185448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,723 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/008d9283618849f08044c0cf76f77e20 2024-11-07T12:38:45,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/a2d20a83814248d6b5eb0a4c4447bcfb is 50, key is test_row_0/B:col10/1730983123599/Put/seqid=0 2024-11-07T12:38:45,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:45,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742485_1661 (size=12301) 2024-11-07T12:38:45,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:45,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:45,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983185945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,949 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bc1940ad44a4ffd987a2f6164472fdb6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983185961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983185961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983185962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983185965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:45,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:46,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:46,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:46,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/a2d20a83814248d6b5eb0a4c4447bcfb 2024-11-07T12:38:46,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/4c20710f04b34d9b8a57ea8539f77e55 is 50, key is test_row_0/C:col10/1730983123599/Put/seqid=0 2024-11-07T12:38:46,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742486_1662 (size=12301) 2024-11-07T12:38:46,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:46,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:46,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:38:46,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:38:46,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/4c20710f04b34d9b8a57ea8539f77e55 2024-11-07T12:38:46,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/008d9283618849f08044c0cf76f77e20 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/008d9283618849f08044c0cf76f77e20 2024-11-07T12:38:46,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/008d9283618849f08044c0cf76f77e20, entries=200, sequenceid=280, filesize=39.0 K 2024-11-07T12:38:46,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/a2d20a83814248d6b5eb0a4c4447bcfb as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a2d20a83814248d6b5eb0a4c4447bcfb 2024-11-07T12:38:46,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a2d20a83814248d6b5eb0a4c4447bcfb, entries=150, sequenceid=280, filesize=12.0 K 2024-11-07T12:38:46,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/4c20710f04b34d9b8a57ea8539f77e55 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/4c20710f04b34d9b8a57ea8539f77e55 2024-11-07T12:38:46,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/4c20710f04b34d9b8a57ea8539f77e55, entries=150, sequenceid=280, filesize=12.0 K 2024-11-07T12:38:46,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for bc1940ad44a4ffd987a2f6164472fdb6 in 1961ms, sequenceid=280, compaction requested=true 2024-11-07T12:38:46,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:46,718 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:46,719 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:46,719 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/A is initiating minor compaction (all files) 2024-11-07T12:38:46,719 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/A in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,719 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2eaf969856824c039259db60436e9f46, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/53a7646b24fc4396b59535097f9b7a12, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/008d9283618849f08044c0cf76f77e20] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=108.7 K 2024-11-07T12:38:46,719 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,719 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2eaf969856824c039259db60436e9f46, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/53a7646b24fc4396b59535097f9b7a12, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/008d9283618849f08044c0cf76f77e20] 2024-11-07T12:38:46,720 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eaf969856824c039259db60436e9f46, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730983121707 2024-11-07T12:38:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:46,721 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53a7646b24fc4396b59535097f9b7a12, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730983122885 2024-11-07T12:38:46,721 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 008d9283618849f08044c0cf76f77e20, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730983123599 2024-11-07T12:38:46,722 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:46,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:46,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:46,729 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:46,729 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/B is initiating minor compaction (all files) 2024-11-07T12:38:46,729 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/B in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,729 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8acb4c7b4b884121b1ea881a661cf99e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/24eb5a1e686a45e2abf3430ff4d2962d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a2d20a83814248d6b5eb0a4c4447bcfb] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=33.9 K 2024-11-07T12:38:46,732 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 8acb4c7b4b884121b1ea881a661cf99e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730983121707 2024-11-07T12:38:46,736 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 24eb5a1e686a45e2abf3430ff4d2962d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730983123533 2024-11-07T12:38:46,740 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d20a83814248d6b5eb0a4c4447bcfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730983123599 2024-11-07T12:38:46,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:38:46,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,757 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:38:46,770 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:46,801 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411071e5a5e272a5d42dca8a726e6c11a4414_bc1940ad44a4ffd987a2f6164472fdb6 store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:46,803 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411071e5a5e272a5d42dca8a726e6c11a4414_bc1940ad44a4ffd987a2f6164472fdb6, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:46,804 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071e5a5e272a5d42dca8a726e6c11a4414_bc1940ad44a4ffd987a2f6164472fdb6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:46,806 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#B#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:46,807 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/2cba4b907e744f42a7242530f6561946 is 50, key is test_row_0/B:col10/1730983123599/Put/seqid=0 2024-11-07T12:38:46,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107149dfc9ddf4545ff94d91ee79b0c7544_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983124785/Put/seqid=0 2024-11-07T12:38:46,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742487_1663 (size=4469) 2024-11-07T12:38:46,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742488_1664 (size=12915) 2024-11-07T12:38:46,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742489_1665 (size=12454) 2024-11-07T12:38:46,939 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/2cba4b907e744f42a7242530f6561946 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/2cba4b907e744f42a7242530f6561946 2024-11-07T12:38:46,985 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/B of bc1940ad44a4ffd987a2f6164472fdb6 into 2cba4b907e744f42a7242530f6561946(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:46,985 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:46,985 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/B, priority=13, startTime=1730983126720; duration=0sec 2024-11-07T12:38:46,985 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:46,985 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:B 2024-11-07T12:38:46,985 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:46,986 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:46,986 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/C is initiating minor compaction (all files) 2024-11-07T12:38:46,986 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/C in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:46,986 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/90727b80dc274242aaa860780be8d049, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/42025a0f3cbd4a418902d09575a4eb38, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/4c20710f04b34d9b8a57ea8539f77e55] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=33.9 K 2024-11-07T12:38:46,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:46,988 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 90727b80dc274242aaa860780be8d049, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730983121707 2024-11-07T12:38:46,989 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 42025a0f3cbd4a418902d09575a4eb38, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730983123533 2024-11-07T12:38:46,989 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c20710f04b34d9b8a57ea8539f77e55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730983123599 2024-11-07T12:38:47,002 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#C#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:47,003 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/a309b0de151c445eb37e741020b05a41 is 50, key is test_row_0/C:col10/1730983123599/Put/seqid=0 2024-11-07T12:38:47,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742490_1666 (size=12915) 2024-11-07T12:38:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983187041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983187043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983187035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983187048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,112 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/a309b0de151c445eb37e741020b05a41 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/a309b0de151c445eb37e741020b05a41 2024-11-07T12:38:47,122 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/C of bc1940ad44a4ffd987a2f6164472fdb6 into a309b0de151c445eb37e741020b05a41(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:47,122 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:47,122 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/C, priority=13, startTime=1730983126720; duration=0sec 2024-11-07T12:38:47,122 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:47,122 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:C 2024-11-07T12:38:47,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983187161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983187161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983187163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983187165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,290 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#A#compaction#556 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:47,291 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/7bf108a76ebb4fb08e38bbe5ac0fa045 is 175, key is test_row_0/A:col10/1730983123599/Put/seqid=0 2024-11-07T12:38:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:47,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742491_1667 (size=31869) 2024-11-07T12:38:47,330 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107149dfc9ddf4545ff94d91ee79b0c7544_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107149dfc9ddf4545ff94d91ee79b0c7544_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:47,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/368859b04d0f4e11afe82abc02b3ae11, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:47,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/368859b04d0f4e11afe82abc02b3ae11 is 175, key is test_row_0/A:col10/1730983124785/Put/seqid=0 2024-11-07T12:38:47,349 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/7bf108a76ebb4fb08e38bbe5ac0fa045 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bf108a76ebb4fb08e38bbe5ac0fa045 2024-11-07T12:38:47,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742492_1668 (size=31255) 2024-11-07T12:38:47,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983187369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983187369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,375 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/368859b04d0f4e11afe82abc02b3ae11 2024-11-07T12:38:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,376 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/A of bc1940ad44a4ffd987a2f6164472fdb6 into 7bf108a76ebb4fb08e38bbe5ac0fa045(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:47,376 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:47,376 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/A, priority=13, startTime=1730983126718; duration=0sec 2024-11-07T12:38:47,376 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:47,376 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:A 2024-11-07T12:38:47,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983187382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983187385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/46d1e6c756364c9ea0b935ca79422d1a is 50, key is test_row_0/B:col10/1730983124785/Put/seqid=0 2024-11-07T12:38:47,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742493_1669 (size=12301) 2024-11-07T12:38:47,453 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/46d1e6c756364c9ea0b935ca79422d1a 2024-11-07T12:38:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/51b6c1f754cb4abea08520273f46a8c3 is 50, key is test_row_0/C:col10/1730983124785/Put/seqid=0 2024-11-07T12:38:47,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742494_1670 (size=12301) 2024-11-07T12:38:47,528 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/51b6c1f754cb4abea08520273f46a8c3 2024-11-07T12:38:47,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/368859b04d0f4e11afe82abc02b3ae11 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/368859b04d0f4e11afe82abc02b3ae11 2024-11-07T12:38:47,593 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/368859b04d0f4e11afe82abc02b3ae11, entries=150, sequenceid=293, filesize=30.5 K 2024-11-07T12:38:47,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/46d1e6c756364c9ea0b935ca79422d1a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/46d1e6c756364c9ea0b935ca79422d1a 2024-11-07T12:38:47,637 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/46d1e6c756364c9ea0b935ca79422d1a, entries=150, sequenceid=293, filesize=12.0 K 2024-11-07T12:38:47,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/51b6c1f754cb4abea08520273f46a8c3 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/51b6c1f754cb4abea08520273f46a8c3 2024-11-07T12:38:47,683 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/51b6c1f754cb4abea08520273f46a8c3, entries=150, sequenceid=293, filesize=12.0 K 2024-11-07T12:38:47,687 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for bc1940ad44a4ffd987a2f6164472fdb6 in 930ms, sequenceid=293, compaction requested=false 2024-11-07T12:38:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-07T12:38:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-07T12:38:47,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-07T12:38:47,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3780 sec 2024-11-07T12:38:47,696 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:47,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:38:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:47,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 3.3860 sec 2024-11-07T12:38:47,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983187709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983187710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983187712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983187717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070a33066d4fb04263b5969b01d78c5640_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983127693/Put/seqid=0 2024-11-07T12:38:47,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742495_1671 (size=14994) 2024-11-07T12:38:47,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983187829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983187829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983187829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983187833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983188038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983188038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983188048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983188048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070a33066d4fb04263b5969b01d78c5640_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070a33066d4fb04263b5969b01d78c5640_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:48,226 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b1cad324f9fe4a5e8142256d8f382019, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:48,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b1cad324f9fe4a5e8142256d8f382019 is 175, key is test_row_0/A:col10/1730983127693/Put/seqid=0 2024-11-07T12:38:48,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742496_1672 (size=39949) 2024-11-07T12:38:48,304 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b1cad324f9fe4a5e8142256d8f382019 2024-11-07T12:38:48,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983188363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983188363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983188363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983188365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/bf1ca2762c784f3a837ba695a648149a is 50, key is test_row_0/B:col10/1730983127693/Put/seqid=0 2024-11-07T12:38:48,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742497_1673 (size=12301) 2024-11-07T12:38:48,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/bf1ca2762c784f3a837ba695a648149a 2024-11-07T12:38:48,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/3d147fb572944589ba409bdfdd9457dc is 50, key is test_row_0/C:col10/1730983127693/Put/seqid=0 2024-11-07T12:38:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742498_1674 (size=12301) 2024-11-07T12:38:48,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/3d147fb572944589ba409bdfdd9457dc 2024-11-07T12:38:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:38:48,493 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:48,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T12:38:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-07T12:38:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:38:48,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/b1cad324f9fe4a5e8142256d8f382019 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b1cad324f9fe4a5e8142256d8f382019 2024-11-07T12:38:48,516 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:38:48,532 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:38:48,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:38:48,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b1cad324f9fe4a5e8142256d8f382019, entries=200, sequenceid=322, filesize=39.0 K 2024-11-07T12:38:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/bf1ca2762c784f3a837ba695a648149a as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bf1ca2762c784f3a837ba695a648149a 2024-11-07T12:38:48,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bf1ca2762c784f3a837ba695a648149a, entries=150, sequenceid=322, filesize=12.0 K 2024-11-07T12:38:48,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/3d147fb572944589ba409bdfdd9457dc as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3d147fb572944589ba409bdfdd9457dc 2024-11-07T12:38:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3d147fb572944589ba409bdfdd9457dc, entries=150, sequenceid=322, filesize=12.0 K 2024-11-07T12:38:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for bc1940ad44a4ffd987a2f6164472fdb6 in 876ms, sequenceid=322, compaction requested=true 2024-11-07T12:38:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:48,572 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:48,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,573 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:48,573 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,573 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/A is initiating minor compaction (all files) 2024-11-07T12:38:48,573 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/A in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:48,574 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bf108a76ebb4fb08e38bbe5ac0fa045, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/368859b04d0f4e11afe82abc02b3ae11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b1cad324f9fe4a5e8142256d8f382019] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=100.7 K 2024-11-07T12:38:48,574 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:48,574 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bf108a76ebb4fb08e38bbe5ac0fa045, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/368859b04d0f4e11afe82abc02b3ae11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b1cad324f9fe4a5e8142256d8f382019] 2024-11-07T12:38:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,574 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bf108a76ebb4fb08e38bbe5ac0fa045, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730983123599 2024-11-07T12:38:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,574 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 368859b04d0f4e11afe82abc02b3ae11, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1730983124771 2024-11-07T12:38:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:38:48,575 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1cad324f9fe4a5e8142256d8f382019, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983127041 2024-11-07T12:38:48,575 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:48,575 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/B is initiating minor compaction (all files) 2024-11-07T12:38:48,575 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/B in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:48,575 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/2cba4b907e744f42a7242530f6561946, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/46d1e6c756364c9ea0b935ca79422d1a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bf1ca2762c784f3a837ba695a648149a] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=36.6 K 2024-11-07T12:38:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,576 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cba4b907e744f42a7242530f6561946, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730983123599 2024-11-07T12:38:48,576 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d1e6c756364c9ea0b935ca79422d1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1730983124771 2024-11-07T12:38:48,576 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1ca2762c784f3a837ba695a648149a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983127041 2024-11-07T12:38:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,589 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,594 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#B#compaction#566 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:48,595 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/356df255e21a4f538eaa0eca2680826e is 50, key is test_row_0/B:col10/1730983127693/Put/seqid=0 2024-11-07T12:38:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,596 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411073a3de5e1206c43e8b7e804c76ea18073_bc1940ad44a4ffd987a2f6164472fdb6 store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,599 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411073a3de5e1206c43e8b7e804c76ea18073_bc1940ad44a4ffd987a2f6164472fdb6, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:48,599 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073a3de5e1206c43e8b7e804c76ea18073_bc1940ad44a4ffd987a2f6164472fdb6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:38:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742499_1675 (size=13017) 2024-11-07T12:38:48,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742500_1676 (size=4469) 2024-11-07T12:38:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,628 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#A#compaction#565 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,629 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/2a84aaef0631444fa9eeffdccee26be9 is 175, key is test_row_0/A:col10/1730983127693/Put/seqid=0 2024-11-07T12:38:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,632 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/356df255e21a4f538eaa0eca2680826e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/356df255e21a4f538eaa0eca2680826e 2024-11-07T12:38:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742501_1677 (size=31971) 2024-11-07T12:38:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,642 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/B of bc1940ad44a4ffd987a2f6164472fdb6 into 356df255e21a4f538eaa0eca2680826e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:48,642 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:48,642 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/B, priority=13, startTime=1730983128572; duration=0sec 2024-11-07T12:38:48,642 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:48,642 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:B 2024-11-07T12:38:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,642 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,643 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:48,643 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/C is initiating minor compaction (all files) 2024-11-07T12:38:48,643 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/C in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:48,644 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/a309b0de151c445eb37e741020b05a41, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/51b6c1f754cb4abea08520273f46a8c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3d147fb572944589ba409bdfdd9457dc] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=36.6 K 2024-11-07T12:38:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,644 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting a309b0de151c445eb37e741020b05a41, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730983123599 2024-11-07T12:38:48,644 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 51b6c1f754cb4abea08520273f46a8c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1730983124771 2024-11-07T12:38:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,645 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d147fb572944589ba409bdfdd9457dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983127041 2024-11-07T12:38:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,646 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/2a84aaef0631444fa9eeffdccee26be9 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2a84aaef0631444fa9eeffdccee26be9 2024-11-07T12:38:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,652 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/A of bc1940ad44a4ffd987a2f6164472fdb6 into 2a84aaef0631444fa9eeffdccee26be9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:48,652 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:48,652 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/A, priority=13, startTime=1730983128572; duration=0sec 2024-11-07T12:38:48,652 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:48,652 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:A 2024-11-07T12:38:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,654 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#C#compaction#567 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,655 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/3702165c9c214c46a3dd2dfb05b7f09d is 50, key is test_row_0/C:col10/1730983127693/Put/seqid=0 2024-11-07T12:38:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742502_1678 (size=13017) 2024-11-07T12:38:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,683 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/3702165c9c214c46a3dd2dfb05b7f09d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3702165c9c214c46a3dd2dfb05b7f09d 2024-11-07T12:38:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,688 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/C of bc1940ad44a4ffd987a2f6164472fdb6 into 3702165c9c214c46a3dd2dfb05b7f09d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:48,688 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:48,688 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/C, priority=13, startTime=1730983128572; duration=0sec 2024-11-07T12:38:48,688 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:48,688 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:C 2024-11-07T12:38:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38515 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-07T12:38:48,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,714 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T12:38:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107064483d0d1e34c2e90627b64d93c2e3c_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983127710/Put/seqid=0 2024-11-07T12:38:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742503_1679 (size=9914) 2024-11-07T12:38:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,755 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107064483d0d1e34c2e90627b64d93c2e3c_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107064483d0d1e34c2e90627b64d93c2e3c_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/aac3e084ba744791a50b60489d2a510e, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:48,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/aac3e084ba744791a50b60489d2a510e is 175, key is test_row_0/A:col10/1730983127710/Put/seqid=0 2024-11-07T12:38:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742504_1680 (size=22561) 2024-11-07T12:38:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:38:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. as already flushing 2024-11-07T12:38:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:48,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983188936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983188937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983188937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:48,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983188945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983189051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983189051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983189051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983189057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:38:49,189 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/aac3e084ba744791a50b60489d2a510e 2024-11-07T12:38:49,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/144f0efbd6e142769def96172c97f7c5 is 50, key is test_row_0/B:col10/1730983127710/Put/seqid=0 2024-11-07T12:38:49,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742505_1681 (size=9857) 2024-11-07T12:38:49,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983189261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983189261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983189265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983189267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983189571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983189571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983189580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983189581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:49,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:38:49,660 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/144f0efbd6e142769def96172c97f7c5 2024-11-07T12:38:49,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/13245e54f7e6442e97b6da2d01f1e80d is 50, key is test_row_0/C:col10/1730983127710/Put/seqid=0 2024-11-07T12:38:49,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742506_1682 (size=9857) 2024-11-07T12:38:49,757 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/13245e54f7e6442e97b6da2d01f1e80d 2024-11-07T12:38:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/aac3e084ba744791a50b60489d2a510e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/aac3e084ba744791a50b60489d2a510e 2024-11-07T12:38:49,817 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/aac3e084ba744791a50b60489d2a510e, entries=100, sequenceid=335, filesize=22.0 K 2024-11-07T12:38:49,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/144f0efbd6e142769def96172c97f7c5 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/144f0efbd6e142769def96172c97f7c5 2024-11-07T12:38:49,824 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/144f0efbd6e142769def96172c97f7c5, entries=100, sequenceid=335, filesize=9.6 K 2024-11-07T12:38:49,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/13245e54f7e6442e97b6da2d01f1e80d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/13245e54f7e6442e97b6da2d01f1e80d 2024-11-07T12:38:49,831 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/13245e54f7e6442e97b6da2d01f1e80d, entries=100, sequenceid=335, filesize=9.6 K 2024-11-07T12:38:49,832 INFO [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for bc1940ad44a4ffd987a2f6164472fdb6 in 1119ms, sequenceid=335, compaction requested=false 2024-11-07T12:38:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/87fb68174c52:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-07T12:38:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-07T12:38:49,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-07T12:38:49,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3140 sec 2024-11-07T12:38:49,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.3790 sec 2024-11-07T12:38:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:50,005 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:50,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T12:38:50,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110716c3f5ecbe6448668934069a9537b496_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983130003/Put/seqid=0 2024-11-07T12:38:50,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983190056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742507_1683 (size=14994) 2024-11-07T12:38:50,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60038 deadline: 1730983190088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:50,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983190088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:50,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60046 deadline: 1730983190088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:50,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60064 deadline: 1730983190101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:50,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983190169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983190379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:50,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:50,494 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110716c3f5ecbe6448668934069a9537b496_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110716c3f5ecbe6448668934069a9537b496_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:50,497 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/acfee4ec8528406da0bddc2467a54ad7, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:50,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/acfee4ec8528406da0bddc2467a54ad7 is 175, key is test_row_0/A:col10/1730983130003/Put/seqid=0 2024-11-07T12:38:50,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742508_1684 (size=39949) 2024-11-07T12:38:50,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/acfee4ec8528406da0bddc2467a54ad7 2024-11-07T12:38:50,636 INFO [Thread-2615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2615. 2024-11-07T12:38:50,636 DEBUG [Thread-2615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:50,636 DEBUG [Thread-2615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,636 INFO [Thread-2619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2619. 2024-11-07T12:38:50,636 DEBUG [Thread-2619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:50,636 DEBUG [Thread-2615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,637 DEBUG [Thread-2619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,637 DEBUG [Thread-2619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:50,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:50,639 INFO [Thread-2617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2617. 2024-11-07T12:38:50,639 DEBUG [Thread-2617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:50,639 DEBUG [Thread-2617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,639 DEBUG [Thread-2617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:50,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/ef7d5e296ba5405fae8d75578e8c5068 is 50, key is test_row_0/B:col10/1730983130003/Put/seqid=0 2024-11-07T12:38:50,641 INFO [Thread-2621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2621. 2024-11-07T12:38:50,641 DEBUG [Thread-2621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:50,641 DEBUG [Thread-2621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,641 DEBUG [Thread-2621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:50,643 INFO [Thread-2613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2613. 2024-11-07T12:38:50,643 DEBUG [Thread-2613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:50,643 DEBUG [Thread-2613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,643 DEBUG [Thread-2613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:50,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:38:50,669 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:50,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742509_1685 (size=12301) 2024-11-07T12:38:50,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/ef7d5e296ba5405fae8d75578e8c5068 2024-11-07T12:38:50,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d6e081ecd26c4508a7aa4682c32e059d is 50, key is test_row_0/C:col10/1730983130003/Put/seqid=0 2024-11-07T12:38:50,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60032 deadline: 1730983190687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:50,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742510_1686 (size=12301) 2024-11-07T12:38:51,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d6e081ecd26c4508a7aa4682c32e059d 2024-11-07T12:38:51,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/acfee4ec8528406da0bddc2467a54ad7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/acfee4ec8528406da0bddc2467a54ad7 2024-11-07T12:38:51,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T12:38:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60020 deadline: 1730983191098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 2024-11-07T12:38:51,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/acfee4ec8528406da0bddc2467a54ad7, entries=200, sequenceid=363, filesize=39.0 K 2024-11-07T12:38:51,099 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 87fb68174c52,38515,1730982961782 for 'test_row_0' in bc1940ad44a4ffd987a2f6164472fdb6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T12:38:51,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/ef7d5e296ba5405fae8d75578e8c5068 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/ef7d5e296ba5405fae8d75578e8c5068 2024-11-07T12:38:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:38:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., hostname=87fb68174c52,38515,1730982961782, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:38:51,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/ef7d5e296ba5405fae8d75578e8c5068, entries=150, sequenceid=363, filesize=12.0 K 2024-11-07T12:38:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/d6e081ecd26c4508a7aa4682c32e059d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d6e081ecd26c4508a7aa4682c32e059d 2024-11-07T12:38:51,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d6e081ecd26c4508a7aa4682c32e059d, entries=150, sequenceid=363, filesize=12.0 K 2024-11-07T12:38:51,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for bc1940ad44a4ffd987a2f6164472fdb6 in 1109ms, sequenceid=363, compaction requested=true 2024-11-07T12:38:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:38:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:38:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:51,114 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bc1940ad44a4ffd987a2f6164472fdb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:38:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:38:51,114 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:51,115 INFO [Thread-2608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2608. 2024-11-07T12:38:51,115 DEBUG [Thread-2608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:51,115 DEBUG [Thread-2608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:51,115 DEBUG [Thread-2608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:51,115 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:51,115 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/B is initiating minor compaction (all files) 2024-11-07T12:38:51,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:51,115 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/B in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:51,116 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/356df255e21a4f538eaa0eca2680826e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/144f0efbd6e142769def96172c97f7c5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/ef7d5e296ba5405fae8d75578e8c5068] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=34.4 K 2024-11-07T12:38:51,116 INFO [Thread-2604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2604. 2024-11-07T12:38:51,116 DEBUG [Thread-2604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:51,116 DEBUG [Thread-2604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:51,116 DEBUG [Thread-2604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:51,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:51,116 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 356df255e21a4f538eaa0eca2680826e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983127041 2024-11-07T12:38:51,116 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:51,116 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/A is initiating minor compaction (all files) 2024-11-07T12:38:51,116 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/A in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:51,116 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2a84aaef0631444fa9eeffdccee26be9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/aac3e084ba744791a50b60489d2a510e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/acfee4ec8528406da0bddc2467a54ad7] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=92.3 K 2024-11-07T12:38:51,117 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:51,117 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. files: [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2a84aaef0631444fa9eeffdccee26be9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/aac3e084ba744791a50b60489d2a510e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/acfee4ec8528406da0bddc2467a54ad7] 2024-11-07T12:38:51,117 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a84aaef0631444fa9eeffdccee26be9, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983127041 2024-11-07T12:38:51,117 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 144f0efbd6e142769def96172c97f7c5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1730983127710 2024-11-07T12:38:51,118 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac3e084ba744791a50b60489d2a510e, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1730983127710 2024-11-07T12:38:51,118 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] compactions.Compactor(225): Compacting acfee4ec8528406da0bddc2467a54ad7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1730983128923 2024-11-07T12:38:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38515 {}] regionserver.HRegion(8855): Flush requested on bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:51,120 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting ef7d5e296ba5405fae8d75578e8c5068, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1730983128923 2024-11-07T12:38:51,120 INFO [Thread-2606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2606. 2024-11-07T12:38:51,120 DEBUG [Thread-2606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:51,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:38:51,120 DEBUG [Thread-2606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:51,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:38:51,120 DEBUG [Thread-2606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:51,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:51,141 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:51,143 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#B#compaction#574 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:51,143 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/5a6004e352614cdd8506bf76cf3d8726 is 50, key is test_row_0/B:col10/1730983130003/Put/seqid=0 2024-11-07T12:38:51,144 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107e3ea6ad9680342739aa5c52906675139_bc1940ad44a4ffd987a2f6164472fdb6 store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:51,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078995f55aa0784f6e9f229189e8dfa991_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983131118/Put/seqid=0 2024-11-07T12:38:51,148 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107e3ea6ad9680342739aa5c52906675139_bc1940ad44a4ffd987a2f6164472fdb6, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:51,148 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e3ea6ad9680342739aa5c52906675139_bc1940ad44a4ffd987a2f6164472fdb6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:51,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742511_1687 (size=13119) 2024-11-07T12:38:51,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742512_1688 (size=12454) 2024-11-07T12:38:51,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742513_1689 (size=4469) 2024-11-07T12:38:51,160 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#A#compaction#575 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:51,160 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/a21b3239c88e42be873ea8bc25e1064e is 175, key is test_row_0/A:col10/1730983130003/Put/seqid=0 2024-11-07T12:38:51,167 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/5a6004e352614cdd8506bf76cf3d8726 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5a6004e352614cdd8506bf76cf3d8726 2024-11-07T12:38:51,172 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/B of bc1940ad44a4ffd987a2f6164472fdb6 into 5a6004e352614cdd8506bf76cf3d8726(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:51,172 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:51,172 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/B, priority=13, startTime=1730983131114; duration=0sec 2024-11-07T12:38:51,172 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:38:51,172 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:B 2024-11-07T12:38:51,172 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:38:51,174 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:38:51,174 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1541): bc1940ad44a4ffd987a2f6164472fdb6/C is initiating minor compaction (all files) 2024-11-07T12:38:51,175 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bc1940ad44a4ffd987a2f6164472fdb6/C in TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:51,175 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3702165c9c214c46a3dd2dfb05b7f09d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/13245e54f7e6442e97b6da2d01f1e80d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d6e081ecd26c4508a7aa4682c32e059d] into tmpdir=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp, totalSize=34.4 K 2024-11-07T12:38:51,175 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 3702165c9c214c46a3dd2dfb05b7f09d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983127041 2024-11-07T12:38:51,175 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting 13245e54f7e6442e97b6da2d01f1e80d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1730983127710 2024-11-07T12:38:51,176 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e081ecd26c4508a7aa4682c32e059d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1730983128923 2024-11-07T12:38:51,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742514_1690 (size=32073) 2024-11-07T12:38:51,188 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc1940ad44a4ffd987a2f6164472fdb6#C#compaction#577 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:38:51,189 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/6f352103e12842b28bee83889f6cc0f1 is 50, key is test_row_0/C:col10/1730983130003/Put/seqid=0 2024-11-07T12:38:51,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742515_1691 (size=13119) 2024-11-07T12:38:51,197 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/6f352103e12842b28bee83889f6cc0f1 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/6f352103e12842b28bee83889f6cc0f1 2024-11-07T12:38:51,199 INFO [Thread-2602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2602. 2024-11-07T12:38:51,199 DEBUG [Thread-2602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:51,199 DEBUG [Thread-2602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:51,199 DEBUG [Thread-2602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:51,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:51,210 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/C of bc1940ad44a4ffd987a2f6164472fdb6 into 6f352103e12842b28bee83889f6cc0f1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:51,210 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:51,210 INFO [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/C, priority=13, startTime=1730983131114; duration=0sec 2024-11-07T12:38:51,210 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:51,210 DEBUG [RS:0;87fb68174c52:38515-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:C 2024-11-07T12:38:51,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:51,562 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078995f55aa0784f6e9f229189e8dfa991_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078995f55aa0784f6e9f229189e8dfa991_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:51,563 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ff29621da4e64b6abe7da3ea0305d2e8, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:51,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ff29621da4e64b6abe7da3ea0305d2e8 is 175, key is test_row_0/A:col10/1730983131118/Put/seqid=0 2024-11-07T12:38:51,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742516_1692 (size=31255) 2024-11-07T12:38:51,586 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/a21b3239c88e42be873ea8bc25e1064e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/a21b3239c88e42be873ea8bc25e1064e 2024-11-07T12:38:51,590 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bc1940ad44a4ffd987a2f6164472fdb6/A of bc1940ad44a4ffd987a2f6164472fdb6 into a21b3239c88e42be873ea8bc25e1064e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:38:51,590 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:51,591 INFO [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., storeName=bc1940ad44a4ffd987a2f6164472fdb6/A, priority=13, startTime=1730983131114; duration=0sec 2024-11-07T12:38:51,591 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:38:51,591 DEBUG [RS:0;87fb68174c52:38515-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc1940ad44a4ffd987a2f6164472fdb6:A 2024-11-07T12:38:51,973 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ff29621da4e64b6abe7da3ea0305d2e8 2024-11-07T12:38:51,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/9bd345f033124d5194544661a1be8bee is 50, key is test_row_0/B:col10/1730983131118/Put/seqid=0 2024-11-07T12:38:51,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742517_1693 (size=12301) 2024-11-07T12:38:51,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/9bd345f033124d5194544661a1be8bee 2024-11-07T12:38:51,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/cca9334330d64205b1edd596ee6b9e9d is 50, key is test_row_0/C:col10/1730983131118/Put/seqid=0 2024-11-07T12:38:52,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742518_1694 (size=12301) 2024-11-07T12:38:52,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/cca9334330d64205b1edd596ee6b9e9d 2024-11-07T12:38:52,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/ff29621da4e64b6abe7da3ea0305d2e8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ff29621da4e64b6abe7da3ea0305d2e8 2024-11-07T12:38:52,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ff29621da4e64b6abe7da3ea0305d2e8, entries=150, sequenceid=374, filesize=30.5 K 2024-11-07T12:38:52,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/9bd345f033124d5194544661a1be8bee as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/9bd345f033124d5194544661a1be8bee 2024-11-07T12:38:52,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/9bd345f033124d5194544661a1be8bee, entries=150, sequenceid=374, filesize=12.0 K 2024-11-07T12:38:52,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/cca9334330d64205b1edd596ee6b9e9d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/cca9334330d64205b1edd596ee6b9e9d 2024-11-07T12:38:52,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/cca9334330d64205b1edd596ee6b9e9d, entries=150, sequenceid=374, filesize=12.0 K 2024-11-07T12:38:52,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=6.71 KB/6870 for bc1940ad44a4ffd987a2f6164472fdb6 in 900ms, sequenceid=374, compaction requested=false 2024-11-07T12:38:52,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bc1940ad44a4ffd987a2f6164472fdb6: 2024-11-07T12:38:53,109 INFO [Thread-2610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2610. 2024-11-07T12:38:53,109 DEBUG [Thread-2610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:38:53,109 DEBUG [Thread-2610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:53,110 DEBUG [Thread-2610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:53,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2301 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2257 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2302 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2310 2024-11-07T12:38:53,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2270 2024-11-07T12:38:53,111 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:38:53,111 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:38:53,111 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:38:53,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:53,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:53,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:53,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T12:38:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:38:53,115 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983133115"}]},"ts":"1730983133115"} 2024-11-07T12:38:53,117 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:38:53,117 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:38:53,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:38:53,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, UNASSIGN}] 2024-11-07T12:38:53,121 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, UNASSIGN 2024-11-07T12:38:53,122 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=bc1940ad44a4ffd987a2f6164472fdb6, regionState=CLOSING, regionLocation=87fb68174c52,38515,1730982961782 2024-11-07T12:38:53,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, UNASSIGN because future has completed 2024-11-07T12:38:53,124 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:38:53,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782}] 2024-11-07T12:38:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:38:53,277 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:53,277 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:38:53,277 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing bc1940ad44a4ffd987a2f6164472fdb6, disabling compactions & flushes 2024-11-07T12:38:53,277 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:53,277 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:53,277 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. after waiting 0 ms 2024-11-07T12:38:53,277 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:53,277 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing bc1940ad44a4ffd987a2f6164472fdb6 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-07T12:38:53,283 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cb5ad98bda7c404792e789ef60dd593a_bc1940ad44a4ffd987a2f6164472fdb6 is 50, key is test_row_0/A:col10/1730983133108/Put/seqid=0 2024-11-07T12:38:53,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742519_1695 (size=9914) 2024-11-07T12:38:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:38:53,693 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:38:53,697 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cb5ad98bda7c404792e789ef60dd593a_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb5ad98bda7c404792e789ef60dd593a_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:53,698 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/418299d2293d4f828ab5c313b6c189a8, store: [table=TestAcidGuarantees family=A region=bc1940ad44a4ffd987a2f6164472fdb6] 2024-11-07T12:38:53,699 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/418299d2293d4f828ab5c313b6c189a8 is 175, key is test_row_0/A:col10/1730983133108/Put/seqid=0 2024-11-07T12:38:53,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742520_1696 (size=22561) 2024-11-07T12:38:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:38:54,105 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/418299d2293d4f828ab5c313b6c189a8 2024-11-07T12:38:54,112 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/c09f0573a8514863ad8ed84ab90e0ff6 is 50, key is test_row_0/B:col10/1730983133108/Put/seqid=0 2024-11-07T12:38:54,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742521_1697 (size=9857) 2024-11-07T12:38:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:38:54,519 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/c09f0573a8514863ad8ed84ab90e0ff6 2024-11-07T12:38:54,525 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/c374f3e1ffb04cbea8ad84f69c2b067b is 50, key is test_row_0/C:col10/1730983133108/Put/seqid=0 2024-11-07T12:38:54,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742522_1698 (size=9857) 2024-11-07T12:38:54,929 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/c374f3e1ffb04cbea8ad84f69c2b067b 2024-11-07T12:38:54,934 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/A/418299d2293d4f828ab5c313b6c189a8 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/418299d2293d4f828ab5c313b6c189a8 2024-11-07T12:38:54,937 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/418299d2293d4f828ab5c313b6c189a8, entries=100, sequenceid=382, filesize=22.0 K 2024-11-07T12:38:54,938 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/B/c09f0573a8514863ad8ed84ab90e0ff6 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/c09f0573a8514863ad8ed84ab90e0ff6 2024-11-07T12:38:54,942 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/c09f0573a8514863ad8ed84ab90e0ff6, entries=100, sequenceid=382, filesize=9.6 K 2024-11-07T12:38:54,942 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/.tmp/C/c374f3e1ffb04cbea8ad84f69c2b067b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c374f3e1ffb04cbea8ad84f69c2b067b 2024-11-07T12:38:54,950 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c374f3e1ffb04cbea8ad84f69c2b067b, entries=100, sequenceid=382, filesize=9.6 K 2024-11-07T12:38:54,951 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for bc1940ad44a4ffd987a2f6164472fdb6 in 1674ms, sequenceid=382, compaction requested=true 2024-11-07T12:38:54,952 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad8ecfccd4244863962fde650bae818a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/af9b509181da4071a219ef918f82318e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/028767d2eb714363b3956fef5581d3ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad5d02921f2e49a2890d329fa64c72d0, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b208e4414a474f27ad0fa0d835163762, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b4200169c5af45e7a475215b6b2d3697, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f5c692cb9132414084ff8807ca51f81b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/94914825e5b24523a72a3f5d393c1c74, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/c8894980c2fb4b4183e43ef8f33342a1, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f6cc518e87c04b2386a3583de90e19be, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/19f2f9e43c584e4d875594bdb9735b0b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/3aeb15c232c64c3aa777a153cf74b22f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f7c7dabb0b784743b64d45e1e439eb97, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bb0c062572641eca8073fc47c8b9136, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/9d067c0b7d084bb3b983e9a9b2b4d93f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2eaf969856824c039259db60436e9f46, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/53a7646b24fc4396b59535097f9b7a12, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/008d9283618849f08044c0cf76f77e20, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bf108a76ebb4fb08e38bbe5ac0fa045, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/368859b04d0f4e11afe82abc02b3ae11, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b1cad324f9fe4a5e8142256d8f382019, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2a84aaef0631444fa9eeffdccee26be9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/aac3e084ba744791a50b60489d2a510e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/acfee4ec8528406da0bddc2467a54ad7] to archive 2024-11-07T12:38:54,953 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:38:54,956 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad8ecfccd4244863962fde650bae818a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad8ecfccd4244863962fde650bae818a 2024-11-07T12:38:54,958 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/af9b509181da4071a219ef918f82318e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/af9b509181da4071a219ef918f82318e 2024-11-07T12:38:54,963 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/028767d2eb714363b3956fef5581d3ac to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/028767d2eb714363b3956fef5581d3ac 2024-11-07T12:38:54,964 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad5d02921f2e49a2890d329fa64c72d0 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ad5d02921f2e49a2890d329fa64c72d0 2024-11-07T12:38:54,966 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b208e4414a474f27ad0fa0d835163762 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b208e4414a474f27ad0fa0d835163762 2024-11-07T12:38:54,968 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b4200169c5af45e7a475215b6b2d3697 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b4200169c5af45e7a475215b6b2d3697 2024-11-07T12:38:54,969 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f5c692cb9132414084ff8807ca51f81b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f5c692cb9132414084ff8807ca51f81b 2024-11-07T12:38:54,971 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/94914825e5b24523a72a3f5d393c1c74 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/94914825e5b24523a72a3f5d393c1c74 2024-11-07T12:38:54,972 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/c8894980c2fb4b4183e43ef8f33342a1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/c8894980c2fb4b4183e43ef8f33342a1 2024-11-07T12:38:54,973 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f6cc518e87c04b2386a3583de90e19be to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f6cc518e87c04b2386a3583de90e19be 2024-11-07T12:38:54,974 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/19f2f9e43c584e4d875594bdb9735b0b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/19f2f9e43c584e4d875594bdb9735b0b 2024-11-07T12:38:54,976 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/3aeb15c232c64c3aa777a153cf74b22f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/3aeb15c232c64c3aa777a153cf74b22f 2024-11-07T12:38:54,983 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f7c7dabb0b784743b64d45e1e439eb97 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/f7c7dabb0b784743b64d45e1e439eb97 2024-11-07T12:38:54,984 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bb0c062572641eca8073fc47c8b9136 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bb0c062572641eca8073fc47c8b9136 2024-11-07T12:38:54,985 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/9d067c0b7d084bb3b983e9a9b2b4d93f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/9d067c0b7d084bb3b983e9a9b2b4d93f 2024-11-07T12:38:54,986 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2eaf969856824c039259db60436e9f46 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2eaf969856824c039259db60436e9f46 2024-11-07T12:38:54,988 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/53a7646b24fc4396b59535097f9b7a12 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/53a7646b24fc4396b59535097f9b7a12 2024-11-07T12:38:54,989 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/008d9283618849f08044c0cf76f77e20 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/008d9283618849f08044c0cf76f77e20 2024-11-07T12:38:54,990 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bf108a76ebb4fb08e38bbe5ac0fa045 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/7bf108a76ebb4fb08e38bbe5ac0fa045 2024-11-07T12:38:54,991 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/368859b04d0f4e11afe82abc02b3ae11 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/368859b04d0f4e11afe82abc02b3ae11 2024-11-07T12:38:54,992 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b1cad324f9fe4a5e8142256d8f382019 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/b1cad324f9fe4a5e8142256d8f382019 2024-11-07T12:38:54,993 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2a84aaef0631444fa9eeffdccee26be9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/2a84aaef0631444fa9eeffdccee26be9 2024-11-07T12:38:54,995 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/aac3e084ba744791a50b60489d2a510e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/aac3e084ba744791a50b60489d2a510e 2024-11-07T12:38:54,996 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/acfee4ec8528406da0bddc2467a54ad7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/acfee4ec8528406da0bddc2467a54ad7 2024-11-07T12:38:54,998 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58f831156d8a458dbaa08c172add31ca, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/6828809ce7b84b9090142eabb6a75a45, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/d1badeeead804229bf1ea4357cdeb68d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/4fabea028d524d26956271ebc4280f6e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/737aef046f134b2eba2b23e4c5621053, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/eb56071977aa4a7b9fc86c5d7ddecf7c, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5a66f1ded79a4e98b5b03f0fc981aa8a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a969ef64a6024803b5de85c6b64001db, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/38c25c1535fb4ae59f6890097d820202, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5da5331ed82f4769b0c16541c3d91e8d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/71cf2ea4b74345dd8f159c2d5f14362e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bb9f94ce3fa341228182ef21a78855c9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8266ef85189a46e49e5db6a275a9d282, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/fbd2aba7c6db47cba023cf53b527af42, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8acb4c7b4b884121b1ea881a661cf99e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58430af085d44f9e87d1a337aa750b97, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/24eb5a1e686a45e2abf3430ff4d2962d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/2cba4b907e744f42a7242530f6561946, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a2d20a83814248d6b5eb0a4c4447bcfb, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/46d1e6c756364c9ea0b935ca79422d1a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/356df255e21a4f538eaa0eca2680826e, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bf1ca2762c784f3a837ba695a648149a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/144f0efbd6e142769def96172c97f7c5, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/ef7d5e296ba5405fae8d75578e8c5068] to archive 2024-11-07T12:38:54,999 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:38:55,001 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58f831156d8a458dbaa08c172add31ca to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58f831156d8a458dbaa08c172add31ca 2024-11-07T12:38:55,002 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/6828809ce7b84b9090142eabb6a75a45 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/6828809ce7b84b9090142eabb6a75a45 2024-11-07T12:38:55,003 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/d1badeeead804229bf1ea4357cdeb68d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/d1badeeead804229bf1ea4357cdeb68d 2024-11-07T12:38:55,004 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/4fabea028d524d26956271ebc4280f6e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/4fabea028d524d26956271ebc4280f6e 2024-11-07T12:38:55,005 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/737aef046f134b2eba2b23e4c5621053 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/737aef046f134b2eba2b23e4c5621053 2024-11-07T12:38:55,006 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/eb56071977aa4a7b9fc86c5d7ddecf7c to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/eb56071977aa4a7b9fc86c5d7ddecf7c 2024-11-07T12:38:55,007 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5a66f1ded79a4e98b5b03f0fc981aa8a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5a66f1ded79a4e98b5b03f0fc981aa8a 2024-11-07T12:38:55,008 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a969ef64a6024803b5de85c6b64001db to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a969ef64a6024803b5de85c6b64001db 2024-11-07T12:38:55,009 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/38c25c1535fb4ae59f6890097d820202 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/38c25c1535fb4ae59f6890097d820202 2024-11-07T12:38:55,010 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5da5331ed82f4769b0c16541c3d91e8d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5da5331ed82f4769b0c16541c3d91e8d 2024-11-07T12:38:55,011 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/71cf2ea4b74345dd8f159c2d5f14362e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/71cf2ea4b74345dd8f159c2d5f14362e 2024-11-07T12:38:55,012 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bb9f94ce3fa341228182ef21a78855c9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bb9f94ce3fa341228182ef21a78855c9 2024-11-07T12:38:55,013 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8266ef85189a46e49e5db6a275a9d282 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8266ef85189a46e49e5db6a275a9d282 2024-11-07T12:38:55,014 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/fbd2aba7c6db47cba023cf53b527af42 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/fbd2aba7c6db47cba023cf53b527af42 2024-11-07T12:38:55,015 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8acb4c7b4b884121b1ea881a661cf99e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/8acb4c7b4b884121b1ea881a661cf99e 2024-11-07T12:38:55,016 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58430af085d44f9e87d1a337aa750b97 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/58430af085d44f9e87d1a337aa750b97 2024-11-07T12:38:55,017 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/24eb5a1e686a45e2abf3430ff4d2962d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/24eb5a1e686a45e2abf3430ff4d2962d 2024-11-07T12:38:55,018 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/2cba4b907e744f42a7242530f6561946 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/2cba4b907e744f42a7242530f6561946 2024-11-07T12:38:55,019 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a2d20a83814248d6b5eb0a4c4447bcfb to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/a2d20a83814248d6b5eb0a4c4447bcfb 2024-11-07T12:38:55,020 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/46d1e6c756364c9ea0b935ca79422d1a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/46d1e6c756364c9ea0b935ca79422d1a 2024-11-07T12:38:55,021 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/356df255e21a4f538eaa0eca2680826e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/356df255e21a4f538eaa0eca2680826e 2024-11-07T12:38:55,023 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bf1ca2762c784f3a837ba695a648149a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/bf1ca2762c784f3a837ba695a648149a 2024-11-07T12:38:55,024 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/144f0efbd6e142769def96172c97f7c5 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/144f0efbd6e142769def96172c97f7c5 2024-11-07T12:38:55,025 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/ef7d5e296ba5405fae8d75578e8c5068 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/ef7d5e296ba5405fae8d75578e8c5068 2024-11-07T12:38:55,026 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/37f4313a45284e3580bbbab46d4828d2, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d10d2628dae14bc584a9000bc3ee5293, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d4c272987ae44ec7b1e8e3a108692a2a, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/5b6c380677524fd3ba1fd5b04bd3cad9, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/296f004b89e449dc92586b85d0ee9375, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/83698e832cc04a408efef182aa3832ae, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c514e949b5dc4901b2e39dae16b04a24, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/2d752af9b2fe46d1a88adabea361616b, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/f27d08f094b54636a3a067a20d6c206f, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59ae84977453405b89400a35e0efd1f7, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/020c4700d3f34d6b86eb0e9c70d34ada, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59c000a9b0a34a8c9164e44e1da3a1e3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/8ecd745d04c7438bbc04e9b437cd19d6, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/72c79940bb4048f999a790800585c3ac, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/90727b80dc274242aaa860780be8d049, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d74bb482f1784f0eae424de5c23f1177, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/42025a0f3cbd4a418902d09575a4eb38, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/a309b0de151c445eb37e741020b05a41, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/4c20710f04b34d9b8a57ea8539f77e55, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/51b6c1f754cb4abea08520273f46a8c3, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3702165c9c214c46a3dd2dfb05b7f09d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3d147fb572944589ba409bdfdd9457dc, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/13245e54f7e6442e97b6da2d01f1e80d, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d6e081ecd26c4508a7aa4682c32e059d] to archive 2024-11-07T12:38:55,027 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:38:55,028 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/37f4313a45284e3580bbbab46d4828d2 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/37f4313a45284e3580bbbab46d4828d2 2024-11-07T12:38:55,029 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d10d2628dae14bc584a9000bc3ee5293 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d10d2628dae14bc584a9000bc3ee5293 2024-11-07T12:38:55,030 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d4c272987ae44ec7b1e8e3a108692a2a to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d4c272987ae44ec7b1e8e3a108692a2a 2024-11-07T12:38:55,031 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/5b6c380677524fd3ba1fd5b04bd3cad9 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/5b6c380677524fd3ba1fd5b04bd3cad9 2024-11-07T12:38:55,033 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/296f004b89e449dc92586b85d0ee9375 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/296f004b89e449dc92586b85d0ee9375 2024-11-07T12:38:55,035 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/83698e832cc04a408efef182aa3832ae to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/83698e832cc04a408efef182aa3832ae 2024-11-07T12:38:55,036 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c514e949b5dc4901b2e39dae16b04a24 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c514e949b5dc4901b2e39dae16b04a24 2024-11-07T12:38:55,037 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/2d752af9b2fe46d1a88adabea361616b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/2d752af9b2fe46d1a88adabea361616b 2024-11-07T12:38:55,038 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/f27d08f094b54636a3a067a20d6c206f to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/f27d08f094b54636a3a067a20d6c206f 2024-11-07T12:38:55,039 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59ae84977453405b89400a35e0efd1f7 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59ae84977453405b89400a35e0efd1f7 2024-11-07T12:38:55,040 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/020c4700d3f34d6b86eb0e9c70d34ada to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/020c4700d3f34d6b86eb0e9c70d34ada 2024-11-07T12:38:55,041 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59c000a9b0a34a8c9164e44e1da3a1e3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/59c000a9b0a34a8c9164e44e1da3a1e3 2024-11-07T12:38:55,042 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/8ecd745d04c7438bbc04e9b437cd19d6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/8ecd745d04c7438bbc04e9b437cd19d6 2024-11-07T12:38:55,043 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/72c79940bb4048f999a790800585c3ac to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/72c79940bb4048f999a790800585c3ac 2024-11-07T12:38:55,045 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/90727b80dc274242aaa860780be8d049 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/90727b80dc274242aaa860780be8d049 2024-11-07T12:38:55,046 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d74bb482f1784f0eae424de5c23f1177 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d74bb482f1784f0eae424de5c23f1177 2024-11-07T12:38:55,047 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/42025a0f3cbd4a418902d09575a4eb38 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/42025a0f3cbd4a418902d09575a4eb38 2024-11-07T12:38:55,048 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/a309b0de151c445eb37e741020b05a41 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/a309b0de151c445eb37e741020b05a41 2024-11-07T12:38:55,049 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/4c20710f04b34d9b8a57ea8539f77e55 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/4c20710f04b34d9b8a57ea8539f77e55 2024-11-07T12:38:55,050 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/51b6c1f754cb4abea08520273f46a8c3 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/51b6c1f754cb4abea08520273f46a8c3 2024-11-07T12:38:55,052 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3702165c9c214c46a3dd2dfb05b7f09d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3702165c9c214c46a3dd2dfb05b7f09d 2024-11-07T12:38:55,053 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3d147fb572944589ba409bdfdd9457dc to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/3d147fb572944589ba409bdfdd9457dc 2024-11-07T12:38:55,054 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/13245e54f7e6442e97b6da2d01f1e80d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/13245e54f7e6442e97b6da2d01f1e80d 2024-11-07T12:38:55,055 DEBUG [StoreCloser-TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d6e081ecd26c4508a7aa4682c32e059d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/d6e081ecd26c4508a7aa4682c32e059d 2024-11-07T12:38:55,069 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/recovered.edits/385.seqid, newMaxSeqId=385, maxSeqId=4 2024-11-07T12:38:55,070 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. 2024-11-07T12:38:55,070 DEBUG [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for bc1940ad44a4ffd987a2f6164472fdb6: Waiting for close lock at 1730983133277Running coprocessor pre-close hooks at 1730983133277Disabling compacts and flushes for region at 1730983133277Disabling writes for close at 1730983133277Obtaining lock to block concurrent updates at 1730983133277Preparing flush snapshotting stores in bc1940ad44a4ffd987a2f6164472fdb6 at 1730983133277Finished memstore snapshotting TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1730983133277Flushing stores of TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6. at 1730983133278 (+1 ms)Flushing bc1940ad44a4ffd987a2f6164472fdb6/A: creating writer at 1730983133278Flushing mob file bc1940ad44a4ffd987a2f6164472fdb6/A: appending metadata at 1730983133283 (+5 ms)Flushing mob file bc1940ad44a4ffd987a2f6164472fdb6/A: closing flushed file at 1730983133283Flushing bc1940ad44a4ffd987a2f6164472fdb6/A: appending metadata at 1730983133698 (+415 ms)Flushing bc1940ad44a4ffd987a2f6164472fdb6/A: closing flushed file at 1730983133698Flushing bc1940ad44a4ffd987a2f6164472fdb6/B: creating writer at 1730983134109 (+411 ms)Flushing bc1940ad44a4ffd987a2f6164472fdb6/B: appending metadata at 1730983134112 (+3 ms)Flushing bc1940ad44a4ffd987a2f6164472fdb6/B: closing flushed file at 1730983134112Flushing bc1940ad44a4ffd987a2f6164472fdb6/C: creating writer at 1730983134522 (+410 ms)Flushing bc1940ad44a4ffd987a2f6164472fdb6/C: appending metadata at 1730983134525 (+3 ms)Flushing bc1940ad44a4ffd987a2f6164472fdb6/C: closing flushed file at 1730983134525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@584ee3c7: reopening flushed file at 1730983134933 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18825e62: reopening flushed file at 1730983134937 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e6d32a3: reopening flushed file at 1730983134942 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for bc1940ad44a4ffd987a2f6164472fdb6 in 1674ms, sequenceid=382, compaction requested=true at 1730983134951 (+9 ms)Writing region close event to WAL at 1730983135059 (+108 ms)Running coprocessor post-close hooks at 1730983135070 (+11 ms)Closed at 1730983135070 2024-11-07T12:38:55,072 INFO [RS_CLOSE_REGION-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,073 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=bc1940ad44a4ffd987a2f6164472fdb6, regionState=CLOSED 2024-11-07T12:38:55,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 because future has completed 2024-11-07T12:38:55,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-07T12:38:55,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure bc1940ad44a4ffd987a2f6164472fdb6, server=87fb68174c52,38515,1730982961782 in 1.9520 sec 2024-11-07T12:38:55,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-07T12:38:55,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bc1940ad44a4ffd987a2f6164472fdb6, UNASSIGN in 1.9580 sec 2024-11-07T12:38:55,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-07T12:38:55,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9620 sec 2024-11-07T12:38:55,083 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983135082"}]},"ts":"1730983135082"} 2024-11-07T12:38:55,084 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:38:55,084 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:38:55,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9720 sec 2024-11-07T12:38:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:38:55,258 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:55,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T12:38:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:55,261 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T12:38:55,262 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:55,268 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,272 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C, FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/recovered.edits] 2024-11-07T12:38:55,274 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/418299d2293d4f828ab5c313b6c189a8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/418299d2293d4f828ab5c313b6c189a8 2024-11-07T12:38:55,276 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/a21b3239c88e42be873ea8bc25e1064e to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/a21b3239c88e42be873ea8bc25e1064e 2024-11-07T12:38:55,277 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ff29621da4e64b6abe7da3ea0305d2e8 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/A/ff29621da4e64b6abe7da3ea0305d2e8 2024-11-07T12:38:55,280 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5a6004e352614cdd8506bf76cf3d8726 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/5a6004e352614cdd8506bf76cf3d8726 2024-11-07T12:38:55,284 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/9bd345f033124d5194544661a1be8bee to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/9bd345f033124d5194544661a1be8bee 2024-11-07T12:38:55,293 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/c09f0573a8514863ad8ed84ab90e0ff6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/B/c09f0573a8514863ad8ed84ab90e0ff6 2024-11-07T12:38:55,295 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/6f352103e12842b28bee83889f6cc0f1 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/6f352103e12842b28bee83889f6cc0f1 2024-11-07T12:38:55,296 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c374f3e1ffb04cbea8ad84f69c2b067b to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/c374f3e1ffb04cbea8ad84f69c2b067b 2024-11-07T12:38:55,297 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/cca9334330d64205b1edd596ee6b9e9d to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/C/cca9334330d64205b1edd596ee6b9e9d 2024-11-07T12:38:55,299 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/recovered.edits/385.seqid to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6/recovered.edits/385.seqid 2024-11-07T12:38:55,300 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/default/TestAcidGuarantees/bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,300 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:38:55,300 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:38:55,301 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:38:55,304 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107064483d0d1e34c2e90627b64d93c2e3c_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107064483d0d1e34c2e90627b64d93c2e3c_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,304 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070a33066d4fb04263b5969b01d78c5640_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070a33066d4fb04263b5969b01d78c5640_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,305 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f8fbfca66e64f3eb629508180ec4094_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f8fbfca66e64f3eb629508180ec4094_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,306 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107149dfc9ddf4545ff94d91ee79b0c7544_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107149dfc9ddf4545ff94d91ee79b0c7544_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,308 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110716c3f5ecbe6448668934069a9537b496_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110716c3f5ecbe6448668934069a9537b496_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,312 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071d1a8e1669644aa7a1020330c53ca44d_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071d1a8e1669644aa7a1020330c53ca44d_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,313 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072fdeaf333bfc41f981baa47fb34e5833_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072fdeaf333bfc41f981baa47fb34e5833_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,314 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073421476c48c643afaa380a128c00fabd_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073421476c48c643afaa380a128c00fabd_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,315 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735fc2621a4b5402e8d4b09e57d10abf7_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735fc2621a4b5402e8d4b09e57d10abf7_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,316 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a2ca1d316714b1eae149b76d18b91d9_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a2ca1d316714b1eae149b76d18b91d9_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,317 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110788d7ffc5182b4dd7884307bad347fb21_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110788d7ffc5182b4dd7884307bad347fb21_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,318 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078995f55aa0784f6e9f229189e8dfa991_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078995f55aa0784f6e9f229189e8dfa991_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,319 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a50340e0258447bbff1abbb01f85f8a_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a50340e0258447bbff1abbb01f85f8a_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,320 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107959d5dd0b8c643ae8ac3d1da2383cf7e_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107959d5dd0b8c643ae8ac3d1da2383cf7e_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,321 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079fe8918552804164a88313351305e3c1_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079fe8918552804164a88313351305e3c1_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,322 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb5ad98bda7c404792e789ef60dd593a_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb5ad98bda7c404792e789ef60dd593a_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,323 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d9fee03cf49d40678386b20543e3fc21_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d9fee03cf49d40678386b20543e3fc21_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,323 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6944feaf3cf4ecd93c1a212b53bc5b5_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6944feaf3cf4ecd93c1a212b53bc5b5_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,324 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f11a7cfc528a48f88146a9932403d4cb_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f11a7cfc528a48f88146a9932403d4cb_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,325 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdeaef24ef4a424ea45a12937cded766_bc1940ad44a4ffd987a2f6164472fdb6 to hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdeaef24ef4a424ea45a12937cded766_bc1940ad44a4ffd987a2f6164472fdb6 2024-11-07T12:38:55,326 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:38:55,329 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:55,334 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:38:55,338 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:38:55,339 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:55,339 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:38:55,339 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983135339"}]},"ts":"9223372036854775807"} 2024-11-07T12:38:55,341 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:38:55,341 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bc1940ad44a4ffd987a2f6164472fdb6, NAME => 'TestAcidGuarantees,,1730983108469.bc1940ad44a4ffd987a2f6164472fdb6.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:38:55,341 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:38:55,341 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983135341"}]},"ts":"9223372036854775807"} 2024-11-07T12:38:55,344 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:38:55,344 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:38:55,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-07T12:38:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T12:38:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:38:55,370 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:38:55,377 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=643 (was 502) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4405 (was 6498) 2024-11-07T12:38:55,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T12:38:55,377 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:38:55,377 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:38:55,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:55,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:55,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:38:55,378 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T12:38:55,378 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=473275785, stopped=false 2024-11-07T12:38:55,379 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=87fb68174c52,36251,1730982961115 2024-11-07T12:38:55,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T12:38:55,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:38:55,382 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T12:38:55,382 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:38:55,383 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:38:55,383 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:38:55,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:55,383 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '87fb68174c52,38515,1730982961782' ***** 2024-11-07T12:38:55,383 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T12:38:55,384 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T12:38:55,384 INFO [RS:0;87fb68174c52:38515 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T12:38:55,384 INFO [RS:0;87fb68174c52:38515 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T12:38:55,384 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(959): stopping server 87fb68174c52,38515,1730982961782 2024-11-07T12:38:55,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T12:38:55,384 INFO [RS:0;87fb68174c52:38515 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T12:38:55,384 INFO [RS:0;87fb68174c52:38515 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;87fb68174c52:38515. 2024-11-07T12:38:55,384 DEBUG [RS:0;87fb68174c52:38515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:38:55,385 DEBUG [RS:0;87fb68174c52:38515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:55,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:38:55,385 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T12:38:55,385 INFO [RS:0;87fb68174c52:38515 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T12:38:55,385 INFO [RS:0;87fb68174c52:38515 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T12:38:55,385 INFO [RS:0;87fb68174c52:38515 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T12:38:55,385 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T12:38:55,385 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:38:55,388 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T12:38:55,388 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-07T12:38:55,388 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:38:55,389 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T12:38:55,389 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T12:38:55,389 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T12:38:55,389 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T12:38:55,389 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T12:38:55,389 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-07T12:38:55,417 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/info/c79b7fa8d425465fbb84415cacebb58d is 95, key is TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0./info:/1730982994622/DeleteFamily/seqid=0 2024-11-07T12:38:55,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742523_1699 (size=6514) 2024-11-07T12:38:55,480 INFO [regionserver/87fb68174c52:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T12:38:55,589 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:38:55,789 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:38:55,821 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/info/c79b7fa8d425465fbb84415cacebb58d 2024-11-07T12:38:55,847 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/ns/1a10e8751786462f816b4c7e126bf944 is 93, key is TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0./ns:/1730982994582/DeleteFamily/seqid=0 2024-11-07T12:38:55,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742524_1700 (size=5906) 2024-11-07T12:38:55,989 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:38:56,189 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:38:56,255 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/ns/1a10e8751786462f816b4c7e126bf944 2024-11-07T12:38:56,263 INFO [regionserver/87fb68174c52:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T12:38:56,263 INFO [regionserver/87fb68174c52:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T12:38:56,285 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/rep_barrier/2537fd0f3d934ce5ab288f19e655c4f7 is 102, key is TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0./rep_barrier:/1730982994582/DeleteFamily/seqid=0 2024-11-07T12:38:56,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742525_1701 (size=6025) 2024-11-07T12:38:56,289 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/rep_barrier/2537fd0f3d934ce5ab288f19e655c4f7 2024-11-07T12:38:56,316 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/table/5a6145cdb58447bfb123ac8bd5a788ee is 96, key is TestAcidGuarantees,,1730982965699.f7b6889d440fde956272a8aab00142f0./table:/1730982994582/DeleteFamily/seqid=0 2024-11-07T12:38:56,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742526_1702 (size=5936) 2024-11-07T12:38:56,345 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/table/5a6145cdb58447bfb123ac8bd5a788ee 2024-11-07T12:38:56,354 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/info/c79b7fa8d425465fbb84415cacebb58d as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/info/c79b7fa8d425465fbb84415cacebb58d 2024-11-07T12:38:56,358 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/info/c79b7fa8d425465fbb84415cacebb58d, entries=12, sequenceid=90, filesize=6.4 K 2024-11-07T12:38:56,359 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/ns/1a10e8751786462f816b4c7e126bf944 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/ns/1a10e8751786462f816b4c7e126bf944 2024-11-07T12:38:56,363 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/ns/1a10e8751786462f816b4c7e126bf944, entries=8, sequenceid=90, filesize=5.8 K 2024-11-07T12:38:56,364 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/rep_barrier/2537fd0f3d934ce5ab288f19e655c4f7 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/rep_barrier/2537fd0f3d934ce5ab288f19e655c4f7 2024-11-07T12:38:56,368 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/rep_barrier/2537fd0f3d934ce5ab288f19e655c4f7, entries=6, sequenceid=90, filesize=5.9 K 2024-11-07T12:38:56,368 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/.tmp/table/5a6145cdb58447bfb123ac8bd5a788ee as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/table/5a6145cdb58447bfb123ac8bd5a788ee 2024-11-07T12:38:56,372 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/table/5a6145cdb58447bfb123ac8bd5a788ee, entries=7, sequenceid=90, filesize=5.8 K 2024-11-07T12:38:56,373 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 984ms, sequenceid=90, compaction requested=false 2024-11-07T12:38:56,390 DEBUG [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:38:56,393 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-07T12:38:56,393 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:38:56,394 INFO [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T12:38:56,394 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730983135389Running coprocessor pre-close hooks at 1730983135389Disabling compacts and flushes for region at 1730983135389Disabling writes for close at 1730983135389Obtaining lock to block concurrent updates at 1730983135389Preparing flush snapshotting stores in 1588230740 at 1730983135389Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1730983135390 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730983135390Flushing 1588230740/info: creating writer at 1730983135391 (+1 ms)Flushing 1588230740/info: appending metadata at 1730983135416 (+25 ms)Flushing 1588230740/info: closing flushed file at 1730983135416Flushing 1588230740/ns: creating writer at 1730983135825 (+409 ms)Flushing 1588230740/ns: appending metadata at 1730983135847 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1730983135847Flushing 1588230740/rep_barrier: creating writer at 1730983136259 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1730983136284 (+25 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1730983136284Flushing 1588230740/table: creating writer at 1730983136293 (+9 ms)Flushing 1588230740/table: appending metadata at 1730983136315 (+22 ms)Flushing 1588230740/table: closing flushed file at 1730983136315Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@115b435f: reopening flushed file at 1730983136353 (+38 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@679896f5: reopening flushed file at 1730983136359 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@443c2526: reopening flushed file at 1730983136363 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bcee8a6: reopening flushed file at 1730983136368 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 984ms, sequenceid=90, compaction requested=false at 1730983136373 (+5 ms)Writing region close event to WAL at 1730983136383 (+10 ms)Running coprocessor post-close hooks at 1730983136393 (+10 ms)Closed at 1730983136393 2024-11-07T12:38:56,394 DEBUG [RS_CLOSE_META-regionserver/87fb68174c52:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T12:38:56,590 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(976): stopping server 87fb68174c52,38515,1730982961782; all regions closed. 2024-11-07T12:38:56,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741834_1010 (size=25344) 2024-11-07T12:38:56,607 DEBUG [RS:0;87fb68174c52:38515 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/oldWALs 2024-11-07T12:38:56,607 INFO [RS:0;87fb68174c52:38515 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 87fb68174c52%2C38515%2C1730982961782.meta:.meta(num 1730982965232) 2024-11-07T12:38:56,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741832_1008 (size=16099853) 2024-11-07T12:38:56,624 DEBUG [RS:0;87fb68174c52:38515 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/oldWALs 2024-11-07T12:38:56,624 INFO [RS:0;87fb68174c52:38515 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 87fb68174c52%2C38515%2C1730982961782:(num 1730982964322) 2024-11-07T12:38:56,624 DEBUG [RS:0;87fb68174c52:38515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:38:56,624 INFO [RS:0;87fb68174c52:38515 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T12:38:56,624 INFO [RS:0;87fb68174c52:38515 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T12:38:56,624 INFO [RS:0;87fb68174c52:38515 {}] hbase.ChoreService(370): Chore service for: regionserver/87fb68174c52:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T12:38:56,625 INFO [RS:0;87fb68174c52:38515 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T12:38:56,625 INFO [regionserver/87fb68174c52:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T12:38:56,626 INFO [RS:0;87fb68174c52:38515 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38515 2024-11-07T12:38:56,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T12:38:56,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/87fb68174c52,38515,1730982961782 2024-11-07T12:38:56,633 INFO [RS:0;87fb68174c52:38515 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T12:38:56,634 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [87fb68174c52,38515,1730982961782] 2024-11-07T12:38:56,636 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/87fb68174c52,38515,1730982961782 already deleted, retry=false 2024-11-07T12:38:56,636 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 87fb68174c52,38515,1730982961782 expired; onlineServers=0 2024-11-07T12:38:56,636 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '87fb68174c52,36251,1730982961115' ***** 2024-11-07T12:38:56,636 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T12:38:56,636 INFO [M:0;87fb68174c52:36251 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T12:38:56,636 INFO [M:0;87fb68174c52:36251 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T12:38:56,636 DEBUG [M:0;87fb68174c52:36251 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T12:38:56,637 DEBUG [M:0;87fb68174c52:36251 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T12:38:56,637 INFO [M:0;87fb68174c52:36251 {}] hbase.ChoreService(370): Chore service for: master/87fb68174c52:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T12:38:56,637 INFO [M:0;87fb68174c52:36251 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T12:38:56,637 DEBUG [M:0;87fb68174c52:36251 {}] master.HMaster(1795): Stopping service threads 2024-11-07T12:38:56,637 INFO [M:0;87fb68174c52:36251 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T12:38:56,637 INFO [M:0;87fb68174c52:36251 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T12:38:56,637 ERROR [M:0;87fb68174c52:36251 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:43681 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:43681,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-07T12:38:56,638 INFO [M:0;87fb68174c52:36251 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T12:38:56,638 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T12:38:56,638 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T12:38:56,638 DEBUG [master/87fb68174c52:0:becomeActiveMaster-HFileCleaner.large.0-1730982963982 {}] cleaner.HFileCleaner(306): Exit Thread[master/87fb68174c52:0:becomeActiveMaster-HFileCleaner.large.0-1730982963982,5,FailOnTimeoutGroup] 2024-11-07T12:38:56,638 DEBUG [master/87fb68174c52:0:becomeActiveMaster-HFileCleaner.small.0-1730982963983 {}] cleaner.HFileCleaner(306): Exit Thread[master/87fb68174c52:0:becomeActiveMaster-HFileCleaner.small.0-1730982963983,5,FailOnTimeoutGroup] 2024-11-07T12:38:56,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T12:38:56,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:38:56,644 DEBUG [M:0;87fb68174c52:36251 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-07T12:38:56,645 DEBUG [M:0;87fb68174c52:36251 {}] master.ActiveMasterManager(353): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-07T12:38:56,647 INFO [M:0;87fb68174c52:36251 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/.lastflushedseqids 2024-11-07T12:38:56,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742527_1703 (size=72) 2024-11-07T12:38:56,668 INFO [M:0;87fb68174c52:36251 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T12:38:56,668 INFO [M:0;87fb68174c52:36251 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T12:38:56,668 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T12:38:56,668 INFO [M:0;87fb68174c52:36251 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:38:56,668 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:38:56,668 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T12:38:56,668 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:38:56,669 INFO [M:0;87fb68174c52:36251 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=775.09 KB heapSize=953.72 KB 2024-11-07T12:38:56,692 DEBUG [M:0;87fb68174c52:36251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/686bdeb756834ade9cb8a43665515087 is 82, key is hbase:meta,,1/info:regioninfo/1730982965313/Put/seqid=0 2024-11-07T12:38:56,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742528_1704 (size=5672) 2024-11-07T12:38:56,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:38:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38515-0x1001a3de74f0001, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:38:56,741 INFO [RS:0;87fb68174c52:38515 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T12:38:56,741 INFO [RS:0;87fb68174c52:38515 {}] regionserver.HRegionServer(1031): Exiting; stopping=87fb68174c52,38515,1730982961782; zookeeper connection closed. 2024-11-07T12:38:56,742 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c6e0c82 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c6e0c82 2024-11-07T12:38:56,742 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T12:38:57,099 INFO [M:0;87fb68174c52:36251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2206 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/686bdeb756834ade9cb8a43665515087 2024-11-07T12:38:57,130 DEBUG [M:0;87fb68174c52:36251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a0adcd0c0ea5430aa30eff57a16d555b is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x1F/proc:d/1730982997799/Put/seqid=0 2024-11-07T12:38:57,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742529_1705 (size=45252) 2024-11-07T12:38:57,152 INFO [M:0;87fb68174c52:36251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=774.54 KB at sequenceid=2206 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a0adcd0c0ea5430aa30eff57a16d555b 2024-11-07T12:38:57,159 INFO [M:0;87fb68174c52:36251 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a0adcd0c0ea5430aa30eff57a16d555b 2024-11-07T12:38:57,195 DEBUG [M:0;87fb68174c52:36251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ff1051a44a7418a89fcecb9164cd74e is 69, key is 87fb68174c52,38515,1730982961782/rs:state/1730982964096/Put/seqid=0 2024-11-07T12:38:57,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073742530_1706 (size=5156) 2024-11-07T12:38:57,213 INFO [M:0;87fb68174c52:36251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2206 (bloomFilter=true), to=hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ff1051a44a7418a89fcecb9164cd74e 2024-11-07T12:38:57,217 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/686bdeb756834ade9cb8a43665515087 as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/686bdeb756834ade9cb8a43665515087 2024-11-07T12:38:57,225 INFO [M:0;87fb68174c52:36251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/686bdeb756834ade9cb8a43665515087, entries=8, sequenceid=2206, filesize=5.5 K 2024-11-07T12:38:57,226 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a0adcd0c0ea5430aa30eff57a16d555b as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a0adcd0c0ea5430aa30eff57a16d555b 2024-11-07T12:38:57,230 INFO [M:0;87fb68174c52:36251 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a0adcd0c0ea5430aa30eff57a16d555b 2024-11-07T12:38:57,230 INFO [M:0;87fb68174c52:36251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a0adcd0c0ea5430aa30eff57a16d555b, entries=176, sequenceid=2206, filesize=44.2 K 2024-11-07T12:38:57,231 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ff1051a44a7418a89fcecb9164cd74e as hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4ff1051a44a7418a89fcecb9164cd74e 2024-11-07T12:38:57,237 INFO [M:0;87fb68174c52:36251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43681/user/jenkins/test-data/8c9ed6cd-b4f4-71ea-f76d-8f1d5b68ff1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4ff1051a44a7418a89fcecb9164cd74e, entries=1, sequenceid=2206, filesize=5.0 K 2024-11-07T12:38:57,238 INFO [M:0;87fb68174c52:36251 {}] regionserver.HRegion(3140): Finished flush of dataSize ~775.09 KB/793696, heapSize ~953.42 KB/976304, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 570ms, sequenceid=2206, compaction requested=false 2024-11-07T12:38:57,253 INFO [M:0;87fb68174c52:36251 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:38:57,253 DEBUG [M:0;87fb68174c52:36251 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730983136668Disabling compacts and flushes for region at 1730983136668Disabling writes for close at 1730983136668Obtaining lock to block concurrent updates at 1730983136669 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730983136669Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=793696, getHeapSize=976544, getOffHeapSize=0, getCellsCount=2407 at 1730983136669Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730983136669Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730983136670 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730983136692 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730983136692Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730983137105 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730983137130 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730983137130Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730983137159 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730983137194 (+35 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730983137195 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b46f62: reopening flushed file at 1730983137217 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b8271ed: reopening flushed file at 1730983137225 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5821279d: reopening flushed file at 1730983137230 (+5 ms)Finished flush of dataSize ~775.09 KB/793696, heapSize ~953.42 KB/976304, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 570ms, sequenceid=2206, compaction requested=false at 1730983137238 (+8 ms)Writing region close event to WAL at 1730983137253 (+15 ms)Closed at 1730983137253 2024-11-07T12:38:57,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45087 is added to blk_1073741830_1006 (size=938309) 2024-11-07T12:38:57,258 INFO [M:0;87fb68174c52:36251 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T12:38:57,258 INFO [M:0;87fb68174c52:36251 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36251 2024-11-07T12:38:57,258 INFO [M:0;87fb68174c52:36251 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T12:38:57,260 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T12:38:57,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:38:57,366 INFO [M:0;87fb68174c52:36251 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T12:38:57,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36251-0x1001a3de74f0000, quorum=127.0.0.1:62093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:38:57,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T12:38:57,377 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T12:38:57,377 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T12:38:57,378 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T12:38:57,378 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/hadoop.log.dir/,STOPPED} 2024-11-07T12:38:57,386 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T12:38:57,386 WARN [BP-1511532250-172.17.0.3-1730982958310 heartbeating to localhost/127.0.0.1:43681 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T12:38:57,386 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T12:38:57,386 WARN [BP-1511532250-172.17.0.3-1730982958310 heartbeating to localhost/127.0.0.1:43681 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1511532250-172.17.0.3-1730982958310 (Datanode Uuid 0ac894f8-e0ee-43f1-9f78-950a1864e70d) service to localhost/127.0.0.1:43681 2024-11-07T12:38:57,387 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/data/data1/current/BP-1511532250-172.17.0.3-1730982958310 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T12:38:57,388 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T12:38:57,388 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/cluster_c872707f-744d-2184-4124-60d3a14b66d4/data/data2/current/BP-1511532250-172.17.0.3-1730982958310 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T12:38:57,405 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T12:38:57,410 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T12:38:57,411 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T12:38:57,411 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T12:38:57,411 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/58516dc4-a44a-0585-af7a-617e760ab991/hadoop.log.dir/,STOPPED} 2024-11-07T12:38:57,439 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T12:38:57,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down